statcatlib 0.10.1__tar.gz → 0.12.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {statcatlib-0.10.1 → statcatlib-0.12.0}/PKG-INFO +132 -85
- {statcatlib-0.10.1 → statcatlib-0.12.0}/README.md +130 -83
- {statcatlib-0.10.1 → statcatlib-0.12.0}/pyproject.toml +9 -47
- {statcatlib-0.10.1 → statcatlib-0.12.0}/pyproject.toml.orig +11 -80
- statcatlib-0.12.0/src/statcatlib/EGM.md +320 -0
- {statcatlib-0.10.1 → statcatlib-0.12.0}/src/statcatlib/__init__.py +33 -24
- statcatlib-0.12.0/src/statcatlib/admin.py +722 -0
- statcatlib-0.12.0/src/statcatlib/bundle.py +502 -0
- statcatlib-0.12.0/src/statcatlib/catalog.py +743 -0
- statcatlib-0.12.0/src/statcatlib/checks.py +73 -0
- statcatlib-0.12.0/src/statcatlib/cli/EGM.md +142 -0
- {statcatlib-0.10.1 → statcatlib-0.12.0}/src/statcatlib/cli/__init__.py +214 -223
- statcatlib-0.12.0/src/statcatlib/diagnostics.py +501 -0
- {statcatlib-0.10.1 → statcatlib-0.12.0}/src/statcatlib/expr.py +1 -1
- statcatlib-0.12.0/src/statcatlib/model/EGM.md +138 -0
- {statcatlib-0.10.1 → statcatlib-0.12.0}/src/statcatlib/model/dataset.py +5 -5
- {statcatlib-0.10.1 → statcatlib-0.12.0}/src/statcatlib/model/skos.py +25 -1
- {statcatlib-0.10.1 → statcatlib-0.12.0}/src/statcatlib/query.py +176 -71
- statcatlib-0.12.0/src/statcatlib/sources/EGM.md +185 -0
- statcatlib-0.12.0/src/statcatlib/sources/ckan.py +706 -0
- {statcatlib-0.10.1 → statcatlib-0.12.0}/src/statcatlib/sources/sdmx.py +143 -91
- statcatlib-0.12.0/src/statcatlib/store/EGM.md +295 -0
- {statcatlib-0.10.1 → statcatlib-0.12.0}/src/statcatlib/store/__init__.py +2 -1
- statcatlib-0.12.0/src/statcatlib/store/inventory.py +397 -0
- {statcatlib-0.10.1 → statcatlib-0.12.0}/src/statcatlib/store/lance.py +317 -184
- {statcatlib-0.10.1 → statcatlib-0.12.0}/src/statcatlib/store/schema.py +113 -25
- {statcatlib-0.10.1 → statcatlib-0.12.0}/src/statcatlib/store/scopes.py +218 -105
- {statcatlib-0.10.1 → statcatlib-0.12.0}/src/statcatlib/store/sentinel.py +10 -12
- {statcatlib-0.10.1 → statcatlib-0.12.0}/src/statcatlib/store/store.py +1290 -907
- statcatlib-0.12.0/src/statcatlib/store/working_copy.py +928 -0
- statcatlib-0.12.0/src/statcatlib/testing.py +79 -0
- {statcatlib-0.10.1 → statcatlib-0.12.0}/src/statcatlib/urn.py +38 -9
- statcatlib-0.10.1/src/statcatlib/bundle.py +0 -886
- statcatlib-0.10.1/src/statcatlib/sources/ckan.py +0 -490
- statcatlib-0.10.1/src/statcatlib/testing.py +0 -72
- {statcatlib-0.10.1 → statcatlib-0.12.0}/LICENSE +0 -0
- {statcatlib-0.10.1 → statcatlib-0.12.0}/src/statcatlib/model/__init__.py +0 -0
- {statcatlib-0.10.1 → statcatlib-0.12.0}/src/statcatlib/model/agents.py +0 -0
- {statcatlib-0.10.1 → statcatlib-0.12.0}/src/statcatlib/model/catalog.py +0 -0
- {statcatlib-0.10.1 → statcatlib-0.12.0}/src/statcatlib/model/classification.py +0 -0
- {statcatlib-0.10.1 → statcatlib-0.12.0}/src/statcatlib/model/distribution.py +0 -0
- {statcatlib-0.10.1 → statcatlib-0.12.0}/src/statcatlib/model/licence.py +0 -0
- {statcatlib-0.10.1 → statcatlib-0.12.0}/src/statcatlib/model/quality.py +0 -0
- {statcatlib-0.10.1 → statcatlib-0.12.0}/src/statcatlib/model/spatial.py +0 -0
- {statcatlib-0.10.1 → statcatlib-0.12.0}/src/statcatlib/model/temporal.py +0 -0
- {statcatlib-0.10.1 → statcatlib-0.12.0}/src/statcatlib/model/types.py +0 -0
- {statcatlib-0.10.1 → statcatlib-0.12.0}/src/statcatlib/py.typed +0 -0
- {statcatlib-0.10.1 → statcatlib-0.12.0}/src/statcatlib/sources/__init__.py +0 -0
- {statcatlib-0.10.1 → statcatlib-0.12.0}/src/statcatlib/store/row.py +0 -0
- {statcatlib-0.10.1 → statcatlib-0.12.0}/src/statcatlib/store/xhtml.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: statcatlib
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.12.0
|
|
4
4
|
Summary: Queryable open data catalog engine for DCAT-AP, StatDCAT-AP, CKAN, and SDMX
|
|
5
5
|
Keywords: sdmx,dcat,open-data,catalog,duckdb,statistical-data,ckan
|
|
6
6
|
Author: gabrielgellner
|
|
@@ -18,7 +18,7 @@ Classifier: Topic :: Internet :: WWW/HTTP :: Indexing/Search
|
|
|
18
18
|
Requires-Dist: attrs>=26.1.0
|
|
19
19
|
Requires-Dist: duckdb>=2.0.0
|
|
20
20
|
Requires-Dist: httpx>=0.28
|
|
21
|
-
Requires-Dist: sdmxlib>=0.
|
|
21
|
+
Requires-Dist: sdmxlib>=0.81.0,<0.82
|
|
22
22
|
Requires-Dist: lancedb>=0.39 ; extra == 'search'
|
|
23
23
|
Requires-Dist: sentence-transformers>=3.0 ; extra == 'search'
|
|
24
24
|
Requires-Dist: typer>=0.15 ; extra == 'search'
|
|
@@ -34,12 +34,19 @@ Description-Content-Type: text/markdown
|
|
|
34
34
|
|
|
35
35
|
# statcatlib
|
|
36
36
|
|
|
37
|
+
[](https://pypi.org/project/statcatlib/)
|
|
38
|
+
[](https://pypi.org/project/statcatlib/)
|
|
39
|
+
[](LICENSE)
|
|
40
|
+
[](https://statcat-suite.github.io/statcatlib/)
|
|
41
|
+
[](https://github.com/astral-sh/uv)
|
|
42
|
+
[](https://github.com/astral-sh/ruff)
|
|
43
|
+
|
|
37
44
|
> **Note:** This library has been written extensively with AI assistance
|
|
38
45
|
> (Claude Code). Users who are not comfortable with AI-generated code should
|
|
39
46
|
> take that into account before adopting it.
|
|
40
47
|
|
|
41
48
|
**statcatlib** is a Python library for building, managing, and querying
|
|
42
|
-
statistical metadata catalogs.
|
|
49
|
+
statistical metadata catalogs.
|
|
43
50
|
|
|
44
51
|
**Documentation:** <https://statcat-suite.github.io/statcatlib/>
|
|
45
52
|
|
|
@@ -58,12 +65,16 @@ import statcatlib as sc
|
|
|
58
65
|
## What it does
|
|
59
66
|
|
|
60
67
|
statcatlib materializes remote catalog and structural metadata into a local
|
|
61
|
-
|
|
62
|
-
|
|
68
|
+
catalog and exposes a fluent Python API for discovery queries — filtering by
|
|
69
|
+
publisher, theme, dimension, code, free text, or provenance lineage.
|
|
70
|
+
|
|
71
|
+
A catalog is a directory of products. Each product is a chain of immutable
|
|
72
|
+
editions, and each edition holds its datasets, its SDMX structures, its
|
|
73
|
+
observations and its search index in one DuckDB database and its DuckLake.
|
|
63
74
|
|
|
64
|
-
**The catalog is a materialized graph.**
|
|
65
|
-
|
|
66
|
-
|
|
75
|
+
**The catalog is a materialized graph.** Ingest pulls the full structural model
|
|
76
|
+
into the edition's database, SQL JOINs are the graph traversal, and a query
|
|
77
|
+
makes no network calls.
|
|
67
78
|
|
|
68
79
|
## Installation
|
|
69
80
|
|
|
@@ -85,24 +96,47 @@ import statcatlib as sc
|
|
|
85
96
|
import statcatlib.query as q
|
|
86
97
|
import sdmxlib as sl
|
|
87
98
|
|
|
88
|
-
# Creates my_catalog/bundle.duckdb and my_catalog/parquet/
|
|
89
|
-
with sc.Catalog("my_catalog").open() as store:
|
|
90
|
-
with sl.RestRegistry(sl.Provider.ESTAT) as reg:
|
|
91
|
-
reg.get(sl.Dataflow.urn_for(agency="ESTAT", id="NAMA_10_GDP"), resolve=True)
|
|
92
|
-
sc.ingest_sdmx(store, reg.registry)
|
|
93
99
|
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
100
|
+
def german_datasets(view: sc.EditionView) -> sc.SearchResult[sc.AggregateDataset]:
|
|
101
|
+
return (
|
|
102
|
+
view.query(sc.AggregateDataset)
|
|
97
103
|
.filter(q.has_code("geo", "DE"))
|
|
98
|
-
.search("GDP")
|
|
99
104
|
.with_facets("themes", "frequency")
|
|
100
105
|
.execute()
|
|
101
106
|
)
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
# Create the catalog and start the product's working copy, its one unsealed edition
|
|
110
|
+
with sc.Catalog.create("my_catalog") as cat:
|
|
111
|
+
wc = cat.product("estat").working_copy()
|
|
112
|
+
with sl.RestRegistry(sl.Provider.ESTAT) as reg:
|
|
113
|
+
reg.get(sl.Dataflow.urn_for(agency="ESTAT", id="NAMA_10_GDP"), resolve=True)
|
|
114
|
+
sc.ingest_sdmx(wc, reg.registry)
|
|
115
|
+
|
|
116
|
+
# wc.view() reads the working copy, writes included
|
|
117
|
+
print(german_datasets(wc.view()).total, "datasets found")
|
|
118
|
+
|
|
119
|
+
# Seal it into an immutable edition, with its search index, and record it live
|
|
120
|
+
wc.seal()
|
|
121
|
+
|
|
122
|
+
# Open the catalog read-only and query the product's live edition, with no network calls
|
|
123
|
+
with sc.Catalog.open("my_catalog") as cat:
|
|
124
|
+
results = german_datasets(cat.product("estat").view())
|
|
102
125
|
print(results.total, "datasets found")
|
|
103
126
|
print(results.facets["themes"])
|
|
104
127
|
```
|
|
105
128
|
|
|
129
|
+
`Catalog.create(path)` creates a catalog, and `Catalog.create(path, exist_ok=True)`
|
|
130
|
+
reopens an existing one for writing. `cat.product(id).working_copy()` starts the
|
|
131
|
+
product's one unsealed edition, a `WorkingCopy`. Write to it with `put`, the
|
|
132
|
+
`ingest_*` functions and `wc.registry`, then `seal()` it, or `discard()` it.
|
|
133
|
+
`Catalog.open(path)` opens a catalog read-only, and `cat.product(id).view()`
|
|
134
|
+
returns an `EditionView` of the product's live edition, which every query below
|
|
135
|
+
runs on. A view stays on its edition. After `cat.refresh()`, call
|
|
136
|
+
`product.view()` again to read a newer one. `statcatlib.testing.temporary_catalog()`
|
|
137
|
+
creates a throwaway catalog for tests and scripts. The on-disk layout is
|
|
138
|
+
specified in [docs/catalog-format.md](docs/catalog-format.md).
|
|
139
|
+
|
|
106
140
|
## Five dataset kinds
|
|
107
141
|
|
|
108
142
|
statcatlib uses a discriminated union of dataset types aligned with DCAT-AP and
|
|
@@ -125,7 +159,7 @@ sc.GeospatialDataset(identifier="boundaries-2024", ..., crs="EPSG:4326")
|
|
|
125
159
|
sc.PublicationDataset(identifier="pub-71-607-x", ..., doi="10.25318/...", authors=[...])
|
|
126
160
|
```
|
|
127
161
|
|
|
128
|
-
All five share the same
|
|
162
|
+
All five share the same write and query API. Use `sc.BaseDataset` as the bound
|
|
129
163
|
for generic code; `sc.Dataset` is a type alias for the full union.
|
|
130
164
|
|
|
131
165
|
## Source connectors
|
|
@@ -136,24 +170,32 @@ for generic code; `sc.Dataset` is a type alias for the full union.
|
|
|
136
170
|
import sdmxlib as sl
|
|
137
171
|
import statcatlib as sc
|
|
138
172
|
|
|
139
|
-
with sc.Catalog("my_catalog")
|
|
140
|
-
with
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
173
|
+
with sc.Catalog.create("my_catalog", exist_ok=True) as cat:
|
|
174
|
+
with cat.product("estat").working_copy() as wc:
|
|
175
|
+
with sl.RestRegistry(sl.Provider.ESTAT) as reg:
|
|
176
|
+
for df_id in ["NAMA_10_GDP", "UNE_RT_M", "PRIC_HPI_IDX"]:
|
|
177
|
+
reg.get(sl.Dataflow.urn_for(agency="ESTAT", id=df_id), resolve=True)
|
|
178
|
+
sc.ingest_sdmx(wc, reg.registry)
|
|
179
|
+
|
|
180
|
+
# Write observations to the edition's DuckLake and bind them to the dataflow
|
|
181
|
+
sc.ingest_data(wc, reg, "ESTAT:NAMA_10_GDP(latest)", measure_dim="na_item")
|
|
182
|
+
wc.seal()
|
|
183
|
+
|
|
184
|
+
# Read them back through the edition's sdmxlib registry
|
|
185
|
+
with sc.Catalog.open("my_catalog") as cat:
|
|
186
|
+
view = cat.product("estat").view()
|
|
187
|
+
ds = view.get(sc.AggregateDataset, "ESTAT:NAMA_10_GDP(1.0)")
|
|
188
|
+
obs = view.registry.obs(sl.Dataflow.parse_urn(ds.sdmx_dataflow_urn))
|
|
189
|
+
wide = obs.pivot(on=ds.measure_dim) # a polars.DataFrame
|
|
147
190
|
```
|
|
148
191
|
|
|
149
192
|
### CKAN (open.canada.ca, data.gov, ...)
|
|
150
193
|
|
|
151
194
|
```python
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
sc.ingest_ckan(store, client, organization="statcan", rows=500)
|
|
195
|
+
with sc.Catalog.create("my_catalog", exist_ok=True) as cat:
|
|
196
|
+
with cat.product("open-canada").working_copy() as wc:
|
|
197
|
+
sc.ingest_ckan(wc, "https://open.canada.ca/data", organization="statcan-statcan", limit=500)
|
|
198
|
+
wc.seal()
|
|
157
199
|
```
|
|
158
200
|
|
|
159
201
|
## Scope-based graph traversal
|
|
@@ -165,50 +207,51 @@ Navigation builds scope objects without executing SQL; only terminal methods
|
|
|
165
207
|
```python
|
|
166
208
|
import statcatlib as sc
|
|
167
209
|
|
|
168
|
-
|
|
210
|
+
cat = sc.Catalog.open("my_catalog")
|
|
211
|
+
view = cat.product("estat").view()
|
|
169
212
|
|
|
170
213
|
# Navigate themes — no SQL until .collect()
|
|
171
|
-
concepts =
|
|
172
|
-
concept =
|
|
214
|
+
concepts = view.themes["statcan"].collect() # ItemList[Concept]
|
|
215
|
+
concept = view.themes["statcan"]["13"].collect() # Concept
|
|
173
216
|
|
|
174
217
|
# Cross-entity navigation — .datasets returns a lazy QueryBuilder
|
|
175
|
-
datasets =
|
|
218
|
+
datasets = view.themes["statcan"]["13"].datasets.collect()
|
|
176
219
|
|
|
177
220
|
# Enrich with sub-traversal expressions (like Polars' pl.col())
|
|
178
|
-
|
|
221
|
+
view.themes["statcan"].enrich(
|
|
179
222
|
n=sc.each("datasets").count(),
|
|
180
223
|
has_data=sc.each("datasets").exists(),
|
|
181
224
|
).collect()
|
|
182
225
|
|
|
183
226
|
# Codelist navigation — sc.urn builds URN strings for you
|
|
184
|
-
codes =
|
|
185
|
-
code =
|
|
227
|
+
codes = view.codelist(sc.urn.codelist("SDMX", "CL_GEO")).collect() # ItemList[Code]
|
|
228
|
+
code = view.codelist(sc.urn.codelist("SDMX", "CL_GEO"))["ON"].collect() # Code
|
|
186
229
|
|
|
187
230
|
# CodelistsScope — parallel to ThemesScope, supports enrich
|
|
188
231
|
# Enriched output includes labels resolved via sdmx.localized_text
|
|
189
|
-
|
|
232
|
+
view.codelists.lang("en").enrich(n=sc.each("datasets").count()).collect()
|
|
190
233
|
# → [{"urn": "...", "label": "Geography", "n": 5}, ...]
|
|
191
234
|
|
|
192
235
|
# Filter codelists by label text (SQL-level, case-insensitive)
|
|
193
|
-
|
|
236
|
+
view.codelists.filter(text_contains="geo").enrich(n=sc.each("datasets").count()).collect()
|
|
194
237
|
|
|
195
|
-
|
|
238
|
+
view.codelist(urn).label("en") # quick name lookup
|
|
196
239
|
|
|
197
240
|
# Enriched per-code output includes code name labels
|
|
198
|
-
|
|
241
|
+
view.codelist(urn).lang("en").enrich(n=sc.each("datasets").count()).collect()
|
|
199
242
|
# → [{"code_id": "ON", "label": "Ontario", "n": 5}, ...]
|
|
200
243
|
|
|
201
244
|
# Batch label resolution — single SQL query for many codes
|
|
202
|
-
|
|
245
|
+
view.codelist(geo_urn).batch_labels(["CA", "US", "DE"])
|
|
203
246
|
# → {"CA": "Canada", "US": "United States", "DE": "Germany"}
|
|
204
247
|
|
|
205
248
|
# Across multiple codelists at once
|
|
206
|
-
|
|
249
|
+
view.codelists.batch_labels([(geo_urn, "CA"), (freq_urn, "A")], lang="en")
|
|
207
250
|
# → {(geo_urn, "CA"): "Canada", (freq_urn, "A"): "Annual"}
|
|
208
251
|
|
|
209
252
|
# Dimension traversal
|
|
210
|
-
dims =
|
|
211
|
-
codelist =
|
|
253
|
+
dims = view.dimensions(ds).collect() # ItemList[DimensionInfo]
|
|
254
|
+
codelist = view.dimensions(ds)["GEO"].codelist # CodelistScope (lazy)
|
|
212
255
|
```
|
|
213
256
|
|
|
214
257
|
Scope classes: `ConceptSchemesScope`, `ConceptSchemeScope`, `ThemesScope`,
|
|
@@ -226,10 +269,10 @@ expression. Reusable across `.enrich()`, `.filter()`, and `.sort_by()`.
|
|
|
226
269
|
import statcatlib.query as q
|
|
227
270
|
|
|
228
271
|
# Field filters (keyword arguments)
|
|
229
|
-
|
|
272
|
+
view.query(sc.AggregateDataset).filter(publisher="ESTAT", status="current").all()
|
|
230
273
|
|
|
231
274
|
# Composable filter objects
|
|
232
|
-
|
|
275
|
+
view.query(sc.AggregateDataset).filter(
|
|
233
276
|
q.has_code("geo", "DE"), # datasets with GEO=DE in their codelist
|
|
234
277
|
q.has_dimensions(["geo", "freq"]), # datasets with both GEO and FREQ dimensions
|
|
235
278
|
).all()
|
|
@@ -237,10 +280,10 @@ store.query(sc.AggregateDataset).filter(
|
|
|
237
280
|
# Hierarchy descent — any geography *under* Canada, strict (CA itself excluded).
|
|
238
281
|
# The applicable Hierarchy is resolved per dataflow; add include_self=True to
|
|
239
282
|
# match the root as well.
|
|
240
|
-
|
|
283
|
+
view.query(sc.AggregateDataset).filter(q.within("REF_AREA", "CA")).all()
|
|
241
284
|
|
|
242
285
|
# Distribution and service filters
|
|
243
|
-
|
|
286
|
+
view.query(sc.OpenDataset).filter(
|
|
244
287
|
q.distribution(format="CSV"),
|
|
245
288
|
q.has_service(endpoint_url="https://..."),
|
|
246
289
|
).all()
|
|
@@ -250,12 +293,12 @@ store.query(sc.OpenDataset).filter(
|
|
|
250
293
|
|
|
251
294
|
```python
|
|
252
295
|
# Agents that publish datasets with code GEO=CA
|
|
253
|
-
|
|
296
|
+
view.query(sc.Agent).filter(
|
|
254
297
|
q.publishes(q.has_code("geo", "CA"), kind="aggregate")
|
|
255
298
|
).all()
|
|
256
299
|
|
|
257
300
|
# Data services serving aggregate datasets
|
|
258
|
-
|
|
301
|
+
view.query(sc.DataService).filter(
|
|
259
302
|
q.serves(kind="aggregate")
|
|
260
303
|
).all()
|
|
261
304
|
```
|
|
@@ -264,18 +307,18 @@ store.query(sc.DataService).filter(
|
|
|
264
307
|
|
|
265
308
|
```python
|
|
266
309
|
# Spatial coverage filter
|
|
267
|
-
|
|
310
|
+
view.query(sc.BaseDataset).filter(q.has_spatial("Canada")).all()
|
|
268
311
|
|
|
269
312
|
# Keyword filter
|
|
270
|
-
|
|
313
|
+
view.query(sc.BaseDataset).filter(q.has_keyword("employment")).all()
|
|
271
314
|
|
|
272
315
|
# Title/description search (case-insensitive ILIKE)
|
|
273
|
-
|
|
274
|
-
|
|
316
|
+
view.query(sc.BaseDataset).filter(q.title_contains("GDP")).all()
|
|
317
|
+
view.query(sc.BaseDataset).filter(q.description_contains("quarterly")).all()
|
|
275
318
|
|
|
276
319
|
# Sort by multilingual title
|
|
277
|
-
|
|
278
|
-
|
|
320
|
+
view.query(sc.BaseDataset).sort_by("title", lang="en").all()
|
|
321
|
+
view.query(sc.BaseDataset).sort_by("title", lang="en", desc=True).all()
|
|
279
322
|
```
|
|
280
323
|
|
|
281
324
|
### Selective relationship loading
|
|
@@ -285,9 +328,9 @@ Use `.include()` to declare exactly which relationships to batch-load — the re
|
|
|
285
328
|
are set to an `UNLOADED` sentinel:
|
|
286
329
|
|
|
287
330
|
```python
|
|
288
|
-
#
|
|
331
|
+
# Load publisher, themes and keywords only: 3 queries instead of ~17
|
|
289
332
|
results = (
|
|
290
|
-
|
|
333
|
+
view.query(sc.BaseDataset)
|
|
291
334
|
.filter(status="published")
|
|
292
335
|
.include("publisher", "themes", "keywords")
|
|
293
336
|
.sort_by("issued", desc=True)
|
|
@@ -295,18 +338,18 @@ results = (
|
|
|
295
338
|
.all()
|
|
296
339
|
)
|
|
297
340
|
|
|
298
|
-
# Explicit full hydration
|
|
299
|
-
results =
|
|
341
|
+
# Explicit full hydration shows the cost at the call site
|
|
342
|
+
results = view.query(sc.BaseDataset).filter(...).full().all()
|
|
300
343
|
|
|
301
344
|
# get() always loads everything — no include() needed
|
|
302
|
-
ds =
|
|
345
|
+
ds = view.get(sc.AggregateDataset, "ESTAT:NAMA_10_GDP(1.0)")
|
|
303
346
|
```
|
|
304
347
|
|
|
305
348
|
Unloaded fields raise `sc.NotLoadedError` on access. Use `sc.is_unloaded(value)`
|
|
306
349
|
to check before accessing:
|
|
307
350
|
|
|
308
351
|
```python
|
|
309
|
-
ds =
|
|
352
|
+
ds = view.query(sc.BaseDataset).include("publisher").first()
|
|
310
353
|
ds.publisher.name # OK
|
|
311
354
|
ds.themes[0] # raises NotLoadedError: 'themes' was not loaded
|
|
312
355
|
|
|
@@ -327,7 +370,7 @@ bypass full object reconstruction — a single SQL query:
|
|
|
327
370
|
```python
|
|
328
371
|
# Row projection — .select() + .rows() returns Row objects (dict subclass)
|
|
329
372
|
rows = (
|
|
330
|
-
|
|
373
|
+
view.query(sc.BaseDataset)
|
|
331
374
|
.filter(q.title_contains("GDP"))
|
|
332
375
|
.sort_by("title", lang="en")
|
|
333
376
|
.limit(3)
|
|
@@ -339,11 +382,11 @@ rows[0].identifier # attribute-style access
|
|
|
339
382
|
rows[0]["title"] # dict-style access — both work
|
|
340
383
|
|
|
341
384
|
# Flat value projection — .scalars() + .values() returns bare values
|
|
342
|
-
ids =
|
|
385
|
+
ids = view.query(sc.BaseDataset).filter(status="current").scalars("identifier").values()
|
|
343
386
|
# → ["EXR", "M1", "UNEMP", ...]
|
|
344
387
|
|
|
345
388
|
# Existence check — no object reconstruction
|
|
346
|
-
if
|
|
389
|
+
if view.query(sc.AggregateDataset).filter(q.has_code("geo", "DE")).exists():
|
|
347
390
|
...
|
|
348
391
|
```
|
|
349
392
|
|
|
@@ -351,11 +394,11 @@ if store.query(sc.AggregateDataset).filter(q.has_code("geo", "DE")).exists():
|
|
|
351
394
|
|
|
352
395
|
```python
|
|
353
396
|
# BM25 search across titles, descriptions, keywords, themes, and dimensions
|
|
354
|
-
results =
|
|
397
|
+
results = view.search("unemployment", limit=20)
|
|
355
398
|
|
|
356
399
|
# Combined search and filter
|
|
357
400
|
results = (
|
|
358
|
-
|
|
401
|
+
view.query(sc.AggregateDataset)
|
|
359
402
|
.filter(publisher="ESTAT")
|
|
360
403
|
.search("labour force")
|
|
361
404
|
.with_facets("themes", "frequency")
|
|
@@ -364,14 +407,14 @@ results = (
|
|
|
364
407
|
print(results.facets["themes"]) # {"Labour": 18, "Economy": 6, ...}
|
|
365
408
|
|
|
366
409
|
# Aggregation counts
|
|
367
|
-
counts =
|
|
410
|
+
counts = view.facets("publisher", "themes", "frequency", "status")
|
|
368
411
|
```
|
|
369
412
|
|
|
370
413
|
### Lineage and provenance (PROV-O)
|
|
371
414
|
|
|
372
415
|
```python
|
|
373
|
-
# Record derivation relationships between datasets
|
|
374
|
-
|
|
416
|
+
# Record derivation relationships between datasets, in a working copy
|
|
417
|
+
wc.add_lineage(
|
|
375
418
|
"lfs-microdata",
|
|
376
419
|
"14100287",
|
|
377
420
|
"aggregated_from",
|
|
@@ -380,30 +423,34 @@ store.add_lineage(
|
|
|
380
423
|
confidence="asserted",
|
|
381
424
|
)
|
|
382
425
|
|
|
383
|
-
# Transitive upstream/downstream traversal — returns QueryBuilder for chaining
|
|
384
|
-
ancestors =
|
|
385
|
-
dependents =
|
|
426
|
+
# Transitive upstream/downstream traversal on a view — returns QueryBuilder for chaining
|
|
427
|
+
ancestors = view.dataset("14100287").upstream(depth=5).collect()
|
|
428
|
+
dependents = view.dataset("CL_GEO").downstream(relationship="uses_classification").collect()
|
|
386
429
|
|
|
387
430
|
# Chain additional filters after traversal
|
|
388
|
-
current =
|
|
431
|
+
current = view.dataset("14100287").upstream().filter(status="current").collect()
|
|
389
432
|
|
|
390
433
|
# Inspect lineage records
|
|
391
|
-
rows =
|
|
434
|
+
rows = view.dataset("14100287").lineage_records(role="target", relationship="aggregated_from")
|
|
392
435
|
```
|
|
393
436
|
|
|
394
437
|
## Architecture
|
|
395
438
|
|
|
396
|
-
|
|
439
|
+
Each edition holds three layers in one DuckDB database and its DuckLake:
|
|
397
440
|
|
|
398
441
|
```
|
|
399
|
-
Discovery layer —
|
|
442
|
+
Discovery layer — statcat schema (dataset, agent, concept_scheme, concept, distribution, lineage)
|
|
400
443
|
Structural layer — sdmxlib tables (dataflows, dsd_components, codes, codelists)
|
|
401
|
-
Observation layer —
|
|
444
|
+
Observation layer — DuckLake tables (actual time-series data)
|
|
402
445
|
```
|
|
403
446
|
|
|
404
|
-
statcatlib owns the discovery layer. sdmxlib owns the structural layer.
|
|
405
|
-
|
|
406
|
-
|
|
447
|
+
statcatlib owns the discovery layer. sdmxlib owns the structural layer. A working
|
|
448
|
+
copy writes both into its edition's database, and queries JOIN freely across
|
|
449
|
+
both. `ingest_data()` writes each dataflow's observations to a DuckLake table
|
|
450
|
+
through `wc.registry.write_observations` and registers an sdmxlib
|
|
451
|
+
`DataflowBinding`, through which `view.registry.obs(flow)` reads them.
|
|
452
|
+
[docs/catalog-format.md](docs/catalog-format.md) specifies the files of a
|
|
453
|
+
catalog and its editions.
|
|
407
454
|
|
|
408
455
|
## Development
|
|
409
456
|
|
|
@@ -415,12 +462,12 @@ just hooks-install
|
|
|
415
462
|
```
|
|
416
463
|
|
|
417
464
|
```bash
|
|
418
|
-
just ci # full gate: index guard + lint + format + docstrings + Any ratchet +
|
|
465
|
+
just ci # full gate: index guard + lint + format + docstrings + prose + Any ratchet + pyrefly + unit and integration tests + strict docs build
|
|
419
466
|
just test # unit tests
|
|
420
467
|
just test-integration # integration tests (no network)
|
|
421
468
|
just test-live # live tests against real SDMX endpoints
|
|
422
469
|
just lint # ruff check
|
|
423
|
-
just typecheck #
|
|
470
|
+
just typecheck # pyrefly
|
|
424
471
|
just check-indexes # only public PyPI is named anywhere in the repo
|
|
425
472
|
just docs # local docs server
|
|
426
473
|
```
|