statcatlib 0.10.0__tar.gz → 0.11.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (36) hide show
  1. {statcatlib-0.10.0 → statcatlib-0.11.0}/PKG-INFO +20 -14
  2. {statcatlib-0.10.0 → statcatlib-0.11.0}/README.md +19 -13
  3. {statcatlib-0.10.0 → statcatlib-0.11.0}/pyproject.toml +19 -38
  4. {statcatlib-0.10.0 → statcatlib-0.11.0}/pyproject.toml.orig +33 -71
  5. {statcatlib-0.10.0 → statcatlib-0.11.0}/src/statcatlib/expr.py +1 -1
  6. {statcatlib-0.10.0 → statcatlib-0.11.0}/src/statcatlib/model/dataset.py +1 -1
  7. {statcatlib-0.10.0 → statcatlib-0.11.0}/src/statcatlib/model/skos.py +7 -1
  8. {statcatlib-0.10.0 → statcatlib-0.11.0}/src/statcatlib/query.py +63 -12
  9. {statcatlib-0.10.0 → statcatlib-0.11.0}/src/statcatlib/sources/sdmx.py +62 -37
  10. {statcatlib-0.10.0 → statcatlib-0.11.0}/src/statcatlib/store/lance.py +9 -9
  11. {statcatlib-0.10.0 → statcatlib-0.11.0}/src/statcatlib/store/scopes.py +28 -7
  12. {statcatlib-0.10.0 → statcatlib-0.11.0}/src/statcatlib/store/sentinel.py +4 -2
  13. {statcatlib-0.10.0 → statcatlib-0.11.0}/src/statcatlib/store/store.py +71 -22
  14. {statcatlib-0.10.0 → statcatlib-0.11.0}/src/statcatlib/urn.py +38 -9
  15. {statcatlib-0.10.0 → statcatlib-0.11.0}/LICENSE +0 -0
  16. {statcatlib-0.10.0 → statcatlib-0.11.0}/src/statcatlib/__init__.py +0 -0
  17. {statcatlib-0.10.0 → statcatlib-0.11.0}/src/statcatlib/bundle.py +0 -0
  18. {statcatlib-0.10.0 → statcatlib-0.11.0}/src/statcatlib/cli/__init__.py +0 -0
  19. {statcatlib-0.10.0 → statcatlib-0.11.0}/src/statcatlib/model/__init__.py +0 -0
  20. {statcatlib-0.10.0 → statcatlib-0.11.0}/src/statcatlib/model/agents.py +0 -0
  21. {statcatlib-0.10.0 → statcatlib-0.11.0}/src/statcatlib/model/catalog.py +0 -0
  22. {statcatlib-0.10.0 → statcatlib-0.11.0}/src/statcatlib/model/classification.py +0 -0
  23. {statcatlib-0.10.0 → statcatlib-0.11.0}/src/statcatlib/model/distribution.py +0 -0
  24. {statcatlib-0.10.0 → statcatlib-0.11.0}/src/statcatlib/model/licence.py +0 -0
  25. {statcatlib-0.10.0 → statcatlib-0.11.0}/src/statcatlib/model/quality.py +0 -0
  26. {statcatlib-0.10.0 → statcatlib-0.11.0}/src/statcatlib/model/spatial.py +0 -0
  27. {statcatlib-0.10.0 → statcatlib-0.11.0}/src/statcatlib/model/temporal.py +0 -0
  28. {statcatlib-0.10.0 → statcatlib-0.11.0}/src/statcatlib/model/types.py +0 -0
  29. {statcatlib-0.10.0 → statcatlib-0.11.0}/src/statcatlib/py.typed +0 -0
  30. {statcatlib-0.10.0 → statcatlib-0.11.0}/src/statcatlib/sources/__init__.py +0 -0
  31. {statcatlib-0.10.0 → statcatlib-0.11.0}/src/statcatlib/sources/ckan.py +0 -0
  32. {statcatlib-0.10.0 → statcatlib-0.11.0}/src/statcatlib/store/__init__.py +0 -0
  33. {statcatlib-0.10.0 → statcatlib-0.11.0}/src/statcatlib/store/row.py +0 -0
  34. {statcatlib-0.10.0 → statcatlib-0.11.0}/src/statcatlib/store/schema.py +0 -0
  35. {statcatlib-0.10.0 → statcatlib-0.11.0}/src/statcatlib/store/xhtml.py +0 -0
  36. {statcatlib-0.10.0 → statcatlib-0.11.0}/src/statcatlib/testing.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: statcatlib
3
- Version: 0.10.0
3
+ Version: 0.11.0
4
4
  Summary: Queryable open data catalog engine for DCAT-AP, StatDCAT-AP, CKAN, and SDMX
5
5
  Keywords: sdmx,dcat,open-data,catalog,duckdb,statistical-data,ckan
6
6
  Author: gabrielgellner
@@ -39,7 +39,7 @@ Description-Content-Type: text/markdown
39
39
  > take that into account before adopting it.
40
40
 
41
41
  **statcatlib** is a Python library for building, managing, and querying
42
- statistical metadata catalogs. Formerly published as `pinaxlib`.
42
+ statistical metadata catalogs.
43
43
 
44
44
  **Documentation:** <https://statcat-suite.github.io/statcatlib/>
45
45
 
@@ -61,9 +61,9 @@ statcatlib materializes remote catalog and structural metadata into a local
61
61
  DuckDB database and exposes a fluent Python API for discovery queries —
62
62
  filtering by publisher, theme, dimension, code, free text, or provenance lineage.
63
63
 
64
- **The catalog is a materialized graph.** Rather than federated queries across
65
- separate REST endpoints, ingest pulls the full structural model into one database.
66
- SQL JOINs are the graph traversal. No network round-trips at query time.
64
+ **The catalog is a materialized graph.** Ingest pulls the full structural model
65
+ into one database, SQL JOINs are the graph traversal, and a query makes no
66
+ network calls.
67
67
 
68
68
  ## Installation
69
69
 
@@ -85,7 +85,7 @@ import statcatlib as sc
85
85
  import statcatlib.query as q
86
86
  import sdmxlib as sl
87
87
 
88
- # Creates my_catalog/bundle.duckdb and my_catalog/parquet/
88
+ # Creates my_catalog/ with meta.duckdb, lake.ducklake and parquet/
89
89
  with sc.Catalog("my_catalog").open() as store:
90
90
  with sl.RestRegistry(sl.Provider.ESTAT) as reg:
91
91
  reg.get(sl.Dataflow.urn_for(agency="ESTAT", id="NAMA_10_GDP"), resolve=True)
@@ -142,8 +142,13 @@ with sc.Catalog("my_catalog").open() as store:
142
142
  reg.get(sl.Dataflow.urn_for(agency="ESTAT", id=df_id), resolve=True)
143
143
  sc.ingest_sdmx(store, reg.registry)
144
144
 
145
- # Stream observation data to Parquet, attach as Distribution
145
+ # Write observations to the bundle's DuckLake and bind them to the dataflow
146
146
  sc.ingest_data(store, reg, "ESTAT:NAMA_10_GDP(latest)", measure_dim="na_item")
147
+
148
+ # Read them back through the co-located sdmxlib registry
149
+ ds = store.get(sc.AggregateDataset, "ESTAT:NAMA_10_GDP(1.0)")
150
+ obs = store.registry.obs(sl.Dataflow.parse_urn(ds.sdmx_dataflow_urn))
151
+ wide = obs.pivot(on=ds.measure_dim) # a polars.DataFrame
147
152
  ```
148
153
 
149
154
  ### CKAN (open.canada.ca, data.gov, ...)
@@ -285,7 +290,7 @@ Use `.include()` to declare exactly which relationships to batch-load — the re
285
290
  are set to an `UNLOADED` sentinel:
286
291
 
287
292
  ```python
288
- # Only load publisher and themes — 3 queries instead of ~17
293
+ # Load publisher, themes and keywords only: 3 queries instead of ~17
289
294
  results = (
290
295
  store.query(sc.BaseDataset)
291
296
  .filter(status="published")
@@ -295,7 +300,7 @@ results = (
295
300
  .all()
296
301
  )
297
302
 
298
- # Explicit full hydration — useful to make the cost visible at the call site
303
+ # Explicit full hydration shows the cost at the call site
299
304
  results = store.query(sc.BaseDataset).filter(...).full().all()
300
305
 
301
306
  # get() always loads everything — no include() needed
@@ -398,12 +403,13 @@ Three layers share one DuckDB database:
398
403
  ```
399
404
  Discovery layer — Catalog (dataset, agent, concept_scheme, concept, distribution, lineage)
400
405
  Structural layer — sdmxlib tables (dataflows, dsd_components, codes, codelists)
401
- Observation layer — Polars / Parquet (actual time-series data)
406
+ Observation layer — DuckLake tables (actual time-series data, under parquet/)
402
407
  ```
403
408
 
404
409
  statcatlib owns the discovery layer. sdmxlib owns the structural layer. Both write
405
- to the same DuckDB connection — queries JOIN freely across both. Parquet files live
406
- alongside the database and are referenced via DCAT `Distribution` records.
410
+ to the same DuckDB connection, and queries JOIN freely across both. `ingest_data()`
411
+ writes each dataflow's observations to a DuckLake table and registers an sdmxlib
412
+ `DataflowBinding`, through which `store.registry.obs(flow)` reads them.
407
413
 
408
414
  ## Development
409
415
 
@@ -415,12 +421,12 @@ just hooks-install
415
421
  ```
416
422
 
417
423
  ```bash
418
- just ci # full gate: index guard + lint + format + docstrings + Any ratchet + both type checkers + unit tests + strict docs build
424
+ just ci # full gate: index guard + lint + format + docstrings + prose + Any ratchet + pyrefly + unit and integration tests + strict docs build
419
425
  just test # unit tests
420
426
  just test-integration # integration tests (no network)
421
427
  just test-live # live tests against real SDMX endpoints
422
428
  just lint # ruff check
423
- just typecheck # basedpyright
429
+ just typecheck # pyrefly
424
430
  just check-indexes # only public PyPI is named anywhere in the repo
425
431
  just docs # local docs server
426
432
  ```
@@ -5,7 +5,7 @@
5
5
  > take that into account before adopting it.
6
6
 
7
7
  **statcatlib** is a Python library for building, managing, and querying
8
- statistical metadata catalogs. Formerly published as `pinaxlib`.
8
+ statistical metadata catalogs.
9
9
 
10
10
  **Documentation:** <https://statcat-suite.github.io/statcatlib/>
11
11
 
@@ -27,9 +27,9 @@ statcatlib materializes remote catalog and structural metadata into a local
27
27
  DuckDB database and exposes a fluent Python API for discovery queries —
28
28
  filtering by publisher, theme, dimension, code, free text, or provenance lineage.
29
29
 
30
- **The catalog is a materialized graph.** Rather than federated queries across
31
- separate REST endpoints, ingest pulls the full structural model into one database.
32
- SQL JOINs are the graph traversal. No network round-trips at query time.
30
+ **The catalog is a materialized graph.** Ingest pulls the full structural model
31
+ into one database, SQL JOINs are the graph traversal, and a query makes no
32
+ network calls.
33
33
 
34
34
  ## Installation
35
35
 
@@ -51,7 +51,7 @@ import statcatlib as sc
51
51
  import statcatlib.query as q
52
52
  import sdmxlib as sl
53
53
 
54
- # Creates my_catalog/bundle.duckdb and my_catalog/parquet/
54
+ # Creates my_catalog/ with meta.duckdb, lake.ducklake and parquet/
55
55
  with sc.Catalog("my_catalog").open() as store:
56
56
  with sl.RestRegistry(sl.Provider.ESTAT) as reg:
57
57
  reg.get(sl.Dataflow.urn_for(agency="ESTAT", id="NAMA_10_GDP"), resolve=True)
@@ -108,8 +108,13 @@ with sc.Catalog("my_catalog").open() as store:
108
108
  reg.get(sl.Dataflow.urn_for(agency="ESTAT", id=df_id), resolve=True)
109
109
  sc.ingest_sdmx(store, reg.registry)
110
110
 
111
- # Stream observation data to Parquet, attach as Distribution
111
+ # Write observations to the bundle's DuckLake and bind them to the dataflow
112
112
  sc.ingest_data(store, reg, "ESTAT:NAMA_10_GDP(latest)", measure_dim="na_item")
113
+
114
+ # Read them back through the co-located sdmxlib registry
115
+ ds = store.get(sc.AggregateDataset, "ESTAT:NAMA_10_GDP(1.0)")
116
+ obs = store.registry.obs(sl.Dataflow.parse_urn(ds.sdmx_dataflow_urn))
117
+ wide = obs.pivot(on=ds.measure_dim) # a polars.DataFrame
113
118
  ```
114
119
 
115
120
  ### CKAN (open.canada.ca, data.gov, ...)
@@ -251,7 +256,7 @@ Use `.include()` to declare exactly which relationships to batch-load — the re
251
256
  are set to an `UNLOADED` sentinel:
252
257
 
253
258
  ```python
254
- # Only load publisher and themes — 3 queries instead of ~17
259
+ # Load publisher, themes and keywords only: 3 queries instead of ~17
255
260
  results = (
256
261
  store.query(sc.BaseDataset)
257
262
  .filter(status="published")
@@ -261,7 +266,7 @@ results = (
261
266
  .all()
262
267
  )
263
268
 
264
- # Explicit full hydration — useful to make the cost visible at the call site
269
+ # Explicit full hydration shows the cost at the call site
265
270
  results = store.query(sc.BaseDataset).filter(...).full().all()
266
271
 
267
272
  # get() always loads everything — no include() needed
@@ -364,12 +369,13 @@ Three layers share one DuckDB database:
364
369
  ```
365
370
  Discovery layer — Catalog (dataset, agent, concept_scheme, concept, distribution, lineage)
366
371
  Structural layer — sdmxlib tables (dataflows, dsd_components, codes, codelists)
367
- Observation layer — Polars / Parquet (actual time-series data)
372
+ Observation layer — DuckLake tables (actual time-series data, under parquet/)
368
373
  ```
369
374
 
370
375
  statcatlib owns the discovery layer. sdmxlib owns the structural layer. Both write
371
- to the same DuckDB connection — queries JOIN freely across both. Parquet files live
372
- alongside the database and are referenced via DCAT `Distribution` records.
376
+ to the same DuckDB connection, and queries JOIN freely across both. `ingest_data()`
377
+ writes each dataflow's observations to a DuckLake table and registers an sdmxlib
378
+ `DataflowBinding`, through which `store.registry.obs(flow)` reads them.
373
379
 
374
380
  ## Development
375
381
 
@@ -381,12 +387,12 @@ just hooks-install
381
387
  ```
382
388
 
383
389
  ```bash
384
- just ci # full gate: index guard + lint + format + docstrings + Any ratchet + both type checkers + unit tests + strict docs build
390
+ just ci # full gate: index guard + lint + format + docstrings + prose + Any ratchet + pyrefly + unit and integration tests + strict docs build
385
391
  just test # unit tests
386
392
  just test-integration # integration tests (no network)
387
393
  just test-live # live tests against real SDMX endpoints
388
394
  just lint # ruff check
389
- just typecheck # basedpyright
395
+ just typecheck # pyrefly
390
396
  just check-indexes # only public PyPI is named anywhere in the repo
391
397
  just docs # local docs server
392
398
  ```
@@ -1,6 +1,6 @@
1
1
  [project]
2
2
  name = "statcatlib"
3
- version = "0.10.0"
3
+ version = "0.11.0"
4
4
  description = "Queryable open data catalog engine for DCAT-AP, StatDCAT-AP, CKAN, and SDMX"
5
5
  license = "Apache-2.0"
6
6
  license-files = ["LICENSE"]
@@ -71,59 +71,40 @@ allow-init-docstring = true
71
71
 
72
72
  [tool.pytest.ini_options]
73
73
  testpaths = ["tests"]
74
+ strict = true
75
+ filterwarnings = ["error"]
74
76
  markers = [
75
77
  "unit: fast, in-process tests with no network I/O",
76
- "docs: doctests and example-script tests",
77
78
  "integration: tests that use real DuckDB and sdmxlib but no network",
78
79
  "live: tests that make real HTTP calls to external SDMX endpoints",
79
80
  ]
80
81
 
81
- [tool.basedpyright]
82
- pythonVersion = "3.14"
83
- typeCheckingMode = "standard"
84
- venvPath = "."
85
- venv = ".venv"
86
- exclude = [
87
- "scripts",
88
- ".venv",
89
- ]
90
- reportMissingTypeArgument = "error"
91
- reportMissingParameterType = "error"
92
- reportUnnecessaryCast = "error"
93
- reportInvalidCast = "error"
94
- reportPropertyTypeMismatch = "error"
95
- reportInconsistentConstructor = "error"
96
- reportUnnecessaryContains = "error"
97
- reportDeprecated = "error"
98
- reportUnsafeMultipleInheritance = "error"
99
- reportTypeCommentUsage = "error"
100
- reportSelfClsParameterName = "error"
101
- reportUnnecessaryTypeIgnoreComment = "error"
102
- reportAny = "none"
103
- reportExplicitAny = "none"
104
- reportUnknownVariableType = "none"
105
-
106
- [[tool.basedpyright.executionEnvironments]]
107
- root = "tests"
108
- extraPaths = ["."]
109
- reportMissingParameterType = "none"
110
- reportTypedDictNotRequiredAccess = "none"
82
+ [tool.coverage.run]
83
+ source = ["src/statcatlib"]
111
84
 
112
- [[tool.basedpyright.executionEnvironments]]
113
- root = "src"
85
+ [tool.coverage.report]
86
+ show_missing = true
87
+ skip_covered = false
88
+ fail_under = 85
89
+ exclude_lines = [
90
+ "pragma: no cover",
91
+ "if TYPE_CHECKING:",
92
+ "raise NotImplementedError",
93
+ "@overload",
94
+ ]
114
95
 
115
96
  [dependency-groups]
116
97
  dev = [
117
- "basedpyright>=1.38.4",
118
98
  "lancedb>=0.39",
119
99
  "mkdocs-material>=9.7.6",
120
100
  "polars>=1.39.3",
121
- "pydoclint>=0.9.1",
122
- "pyrefly>=1.2.0",
101
+ "pydoclint>=0.11.1,<0.12",
102
+ "pyrefly>=1.4.0,<1.5",
123
103
  "pytest>=9.0.3",
104
+ "pytest-cov>=7.1.0",
124
105
  "pytest-httpx>=0.35",
125
106
  "rich>=15.0",
126
- "ruff>=0.15.8",
107
+ "ruff>=0.17.1,<0.18",
127
108
  "sentence-transformers>=3.0",
128
109
  "typer>=0.15",
129
110
  "watchfiles>=1.1.1",
@@ -1,6 +1,6 @@
1
1
  [project]
2
2
  name = "statcatlib"
3
- version = "0.10.0"
3
+ version = "0.11.0"
4
4
  description = "Queryable open data catalog engine for DCAT-AP, StatDCAT-AP, CKAN, and SDMX"
5
5
  license = "Apache-2.0"
6
6
  license-files = ["LICENSE"]
@@ -74,91 +74,53 @@ allow-init-docstring = true
74
74
 
75
75
  [tool.pytest.ini_options]
76
76
  testpaths = ["tests"]
77
+ # `strict` makes an unregistered marker a collection error: a typo such as
78
+ # `pytest.mark.uint` would otherwise drop a whole file out of `just ci`, which
79
+ # selects by marker (GH-8). It also rejects duplicate parametrize ids.
80
+ strict = true
81
+ # A warning that cannot fail is a warning nobody reads. The suite raised seven
82
+ # when this went on, all statcatlib's own: each test now asserts the warning
83
+ # its setup provokes, or avoids it as a real caller would. No ignore list.
84
+ filterwarnings = ["error"]
77
85
  markers = [
78
86
  "unit: fast, in-process tests with no network I/O",
79
- "docs: doctests and example-script tests",
80
87
  "integration: tests that use real DuckDB and sdmxlib but no network",
81
88
  "live: tests that make real HTTP calls to external SDMX endpoints",
82
89
  ]
83
90
 
91
+ [tool.coverage.run]
92
+ source = ["src/statcatlib"]
93
+
94
+ [tool.coverage.report]
95
+ show_missing = true
96
+ skip_covered = false
97
+ # 88% when this was set (2026-10-23, `just coverage`). A floor, a little under
98
+ # that, so a change that drops a module's tests fails here. `sources/sdmx.py`
99
+ # is the lowest at 54%: `ingest_data` is reached only by the `live` tier.
100
+ fail_under = 85
101
+ exclude_lines = [
102
+ "pragma: no cover",
103
+ "if TYPE_CHECKING:",
104
+ "raise NotImplementedError",
105
+ "@overload",
106
+ ]
107
+
84
108
  [dependency-groups]
85
109
  dev = [
86
- "basedpyright>=1.38.4",
87
110
  "lancedb>=0.39",
88
111
  "mkdocs-material>=9.7.6",
89
112
  "polars>=1.39.3",
90
- "pydoclint>=0.9.1",
91
- "pyrefly>=1.2.0",
113
+ # The checkers are capped, as sdmxlib's are: uv.lock is gitignored, so an
114
+ # uncapped checker's next release would change what `just ci` reports
115
+ # without a commit. A bump goes in its own PR, with its findings fixed.
116
+ "pydoclint>=0.11.1,<0.12",
117
+ "pyrefly>=1.4.0,<1.5",
92
118
  "pytest>=9.0.3",
119
+ "pytest-cov>=7.1.0",
93
120
  "pytest-httpx>=0.35",
94
121
  "rich>=15.0",
95
- "ruff>=0.15.8",
122
+ "ruff>=0.17.1,<0.18",
96
123
  "sentence-transformers>=3.0",
97
124
  "typer>=0.15",
98
125
  "watchfiles>=1.1.1",
99
126
  ]
100
-
101
- [tool.basedpyright]
102
- pythonVersion = "3.14"
103
- typeCheckingMode = "standard"
104
- venvPath = "."
105
- venv = ".venv"
106
- exclude = ["scripts", ".venv"]
107
- # Strict-mode rules that are clean on this codebase (0 findings each, measured
108
- # on the commit that turned them on), enabled as regression guards rather than
109
- # as a cleanup. Borrowed from sdmxlib, which runs the same set — see #106.
110
- #
111
- # The first two are basedpyright's Any-degradation catchers: a bare generic
112
- # (`list` rather than `list[str]`) and an untyped parameter both silently
113
- # become Any.
114
- reportMissingTypeArgument = "error"
115
- reportMissingParameterType = "error"
116
- # Dead casts, getter/setter drift, __init__/__new__ disagreement, and
117
- # always-true/false `in` checks. Pure hardening.
118
- reportUnnecessaryCast = "error"
119
- reportInvalidCast = "error"
120
- reportPropertyTypeMismatch = "error"
121
- reportInconsistentConstructor = "error"
122
- reportUnnecessaryContains = "error"
123
- reportDeprecated = "error"
124
- reportUnsafeMultipleInheritance = "error"
125
- reportTypeCommentUsage = "error"
126
- reportSelfClsParameterName = "error"
127
- # Suppression hygiene, and the rule that keeps the .claude suppression guard
128
- # honest in the other direction: an ignore that outlived its defect is a lie
129
- # the next reader believes. Found 6 the day it went on, none of which were
130
- # hiding anything.
131
- reportUnnecessaryTypeIgnoreComment = "error"
132
- #
133
- # NOT enabled: reportCallInDefaultInitializer (23 findings). `ruff.toml`
134
- # already ignores B008, which is the same concern — turning this on would
135
- # contradict a decision the project has already made. Recorded here so the
136
- # next person diffing against sdmxlib's config does not re-litigate it.
137
- reportAny = "none"
138
- reportExplicitAny = "none"
139
- reportUnknownVariableType = "none"
140
-
141
- # `tests/` is type-checked too (#116) — the gap let a wrong `tmp_path`
142
- # annotation stand behind 25 suppressions, and hid three dead ignores from
143
- # `reportUnnecessaryTypeIgnoreComment`, which exists to catch exactly those.
144
- #
145
- # Annotation-completeness rules stay off there. `ruff.toml` already exempts
146
- # tests from `ANN`; demanding parameter types here would contradict that
147
- # decision rather than extend it. Measured: 285 of basedpyright's 416
148
- # findings over `tests/` were this one rule, and neither bug the gap hid was
149
- # an annotation-coverage finding.
150
- [[tool.basedpyright.executionEnvironments]]
151
- root = "tests"
152
- # `from tests.conftest import …` is how the suite shares fixtures; without the
153
- # repo root on the path the package does not resolve from inside `tests`.
154
- extraPaths = ["."]
155
- reportMissingParameterType = "none"
156
- # 48 — a test collects rows of one kind and reads that kind's keys, which
157
- # `SearchRow` cannot express: every extension key is `NotRequired` because it
158
- # belongs to one row kind, deliberately and per its docstring. The rule is
159
- # right in `src`, where the kind is not known locally, and cannot see the
160
- # narrowing a test performs by construction.
161
- reportTypedDictNotRequiredAccess = "none"
162
-
163
- [[tool.basedpyright.executionEnvironments]]
164
- root = "src"
@@ -5,7 +5,7 @@ any SQL. They are context-free objects — like ``pl.col()`` in Polars — and
5
5
  can be reused across ``.enrich()``, ``.filter()``, and ``.sort_by()`` on any
6
6
  collection scope.
7
7
 
8
- Usage::
8
+ Usage:
9
9
 
10
10
  import statcatlib as sc
11
11
 
@@ -119,7 +119,7 @@ class AggregateDataset(BaseDataset):
119
119
  num_series: int | None = None
120
120
  # SDMX link: FK into sdmxlib's dataflows.urn in the shared DuckDB
121
121
  sdmx_dataflow_urn: str | None = None
122
- # Pivot axis dimension for ingest_data(). None = auto-detect from DSD.
122
+ # Default pivot axis for readers, recorded by ingest_data(). None = the DSD's MeasureDimension.
123
123
  measure_dim: str | None = None
124
124
 
125
125
 
@@ -134,8 +134,14 @@ def is_concept_in_scheme(scheme: ConceptScheme, notation: str) -> bool:
134
134
  Pure-Python — no DuckDB or any other storage involved.
135
135
  Useful for validation pipelines that operate on in-memory domain objects.
136
136
 
137
- Example::
137
+ Args:
138
+ scheme: The concept scheme.
139
+ notation: The concept's notation.
138
140
 
141
+ Returns:
142
+ Whether *scheme* lists *notation* among its concepts.
143
+
144
+ Example:
139
145
  valid = is_concept_in_scheme(product_type_scheme, dataset.dataset_type.notation)
140
146
  """
141
147
  return notation in scheme.concepts
@@ -1,6 +1,6 @@
1
1
  """Composable filter objects for Catalog queries.
2
2
 
3
- Usage::
3
+ Usage:
4
4
 
5
5
  import statcatlib.query as q
6
6
 
@@ -630,8 +630,7 @@ def within(
630
630
  ``q.hcode.ancestors``, which excludes the node itself.
631
631
  codelist_urn: Optional codelist URN to disambiguate same-named dimensions across DSDs.
632
632
 
633
- Example::
634
-
633
+ Example:
635
634
  store.query(sc.AggregateDataset).filter(
636
635
  q.within("REF_AREA", "ON", hierarchy=h_geo_statistical_urn)
637
636
  )
@@ -653,8 +652,13 @@ def has_dimensions(dims: list[str]) -> SdmxDimensionFilter | _NoOpFilter:
653
652
  JOINs through ``sdmx.dataflow → sdmx.dsd_component``.
654
653
  An empty list matches all datasets.
655
654
 
656
- Example::
655
+ Args:
656
+ dims: The dimension ids every matching DSD must have.
657
+
658
+ Returns:
659
+ The filter, or one that matches everything when *dims* is empty.
657
660
 
661
+ Example:
658
662
  store.query(sc.AggregateDataset).filter(q.has_dimensions(["GEO", "SEX"])).execute()
659
663
  """
660
664
  if not dims:
@@ -676,8 +680,14 @@ def distribution(**kwargs: object) -> ExistsFilter:
676
680
  .filter(q.distribution(format="CSV"))
677
681
  .filter(q.distribution(format="JSON"))
678
682
 
679
- Example::
683
+ Args:
684
+ **kwargs: Column values one distribution must all match: ``format``,
685
+ ``media_type``, ``service_type`` or ``endpoint_url``.
686
+
687
+ Returns:
688
+ An EXISTS filter over the dataset's distributions.
680
689
 
690
+ Example:
681
691
  store.query(sc.BaseDataset).filter(q.distribution(media_type="text/csv")).execute()
682
692
  """
683
693
  service_keys = {"service_type", "endpoint_url"}
@@ -709,8 +719,13 @@ def has_service(**kwargs: object) -> ExistsFilter:
709
719
 
710
720
  Supported keys: ``service_type``, ``endpoint_url``.
711
721
 
712
- Example::
722
+ Args:
723
+ **kwargs: Column values the data service must all match.
724
+
725
+ Returns:
726
+ An EXISTS filter over the dataset's distributions and their services.
713
727
 
728
+ Example:
714
729
  store.query(sc.BaseDataset).filter(
715
730
  q.has_service(endpoint_url="https://example.com/sdmx")
716
731
  ).execute()
@@ -849,8 +864,14 @@ def publishes(*inner_filters: PositionalFilter, **kwargs: object) -> ExistsFilte
849
864
  positional arguments, and/or dataset-level conditions as keyword arguments
850
865
  (``kind``, ``status``).
851
866
 
852
- Example::
867
+ Args:
868
+ *inner_filters: Filters a published dataset must match.
869
+ **kwargs: Dataset columns a published dataset must match.
853
870
 
871
+ Returns:
872
+ An EXISTS filter over the agent's datasets.
873
+
874
+ Example:
854
875
  # Agents that publish aggregate datasets about Ontario
855
876
  store.query(sc.Agent).filter(
856
877
  q.publishes(q.has_code("GEO", "ON"), kind="aggregate")
@@ -865,8 +886,14 @@ def serves(*inner_filters: PositionalFilter, **kwargs: object) -> ExistsFilter:
865
886
  Pass ``ExistsFilter`` objects as positional arguments and/or dataset-level
866
887
  conditions as keyword arguments (``kind``, ``status``).
867
888
 
868
- Example::
889
+ Args:
890
+ *inner_filters: Filters a served dataset must match.
891
+ **kwargs: Dataset columns a served dataset must match.
892
+
893
+ Returns:
894
+ An EXISTS filter over the service's datasets.
869
895
 
896
+ Example:
870
897
  # DataServices that serve datasets with a GEO dimension
871
898
  store.query(sc.DataService).filter(q.serves(q.has_dimensions(["GEO"]))).all()
872
899
  """
@@ -881,8 +908,14 @@ def serves(*inner_filters: PositionalFilter, **kwargs: object) -> ExistsFilter:
881
908
  def has_spatial(label: str, *, lang: str = "en") -> ExistsFilter:
882
909
  """Filter datasets that have a spatial coverage matching *label*.
883
910
 
884
- Example::
911
+ Args:
912
+ label: The spatial coverage label, matched exactly.
913
+ lang: The language of *label*.
885
914
 
915
+ Returns:
916
+ An EXISTS filter over the dataset's spatial coverage.
917
+
918
+ Example:
886
919
  store.query(sc.BaseDataset).filter(q.has_spatial("Canada")).execute()
887
920
  """
888
921
  return ExistsFilter(
@@ -894,8 +927,14 @@ def has_spatial(label: str, *, lang: str = "en") -> ExistsFilter:
894
927
  def has_keyword(keyword: str, *, lang: str = "en") -> ExistsFilter:
895
928
  """Filter datasets that have a keyword matching *keyword*.
896
929
 
897
- Example::
930
+ Args:
931
+ keyword: The keyword, matched exactly.
932
+ lang: The language of *keyword*.
933
+
934
+ Returns:
935
+ An EXISTS filter over the dataset's keywords.
898
936
 
937
+ Example:
899
938
  store.query(sc.BaseDataset).filter(q.has_keyword("employment")).execute()
900
939
  """
901
940
  return ExistsFilter(
@@ -907,8 +946,14 @@ def has_keyword(keyword: str, *, lang: str = "en") -> ExistsFilter:
907
946
  def title_contains(text: str, *, lang: str = "en") -> MapFilter:
908
947
  """Filter datasets whose title contains *text* (case-insensitive).
909
948
 
910
- Example::
949
+ Args:
950
+ text: The text to find, anywhere in the title.
951
+ lang: Which language's title to search.
952
+
953
+ Returns:
954
+ A filter on the dataset's title in *lang*.
911
955
 
956
+ Example:
912
957
  store.query(sc.BaseDataset).filter(q.title_contains("GDP")).execute()
913
958
  """
914
959
  return MapFilter(column="title", key=lang, value=text, op="ilike")
@@ -917,8 +962,14 @@ def title_contains(text: str, *, lang: str = "en") -> MapFilter:
917
962
  def description_contains(text: str, *, lang: str = "en") -> MapFilter:
918
963
  """Filter datasets whose description contains *text* (case-insensitive).
919
964
 
920
- Example::
965
+ Args:
966
+ text: The text to find, anywhere in the description.
967
+ lang: Which language's description to search.
968
+
969
+ Returns:
970
+ A filter on the dataset's description in *lang*.
921
971
 
972
+ Example:
922
973
  store.query(sc.BaseDataset).filter(q.description_contains("exchange")).execute()
923
974
  """
924
975
  return MapFilter(column="description", key=lang, value=text, op="ilike")
@@ -6,7 +6,7 @@ This is the only statcatlib module that imports sdmxlib beyond InternationalStri
6
6
  from __future__ import annotations
7
7
 
8
8
  import re
9
- from typing import TYPE_CHECKING
9
+ from typing import TYPE_CHECKING, Protocol
10
10
 
11
11
  from sdmxlib import InternationalString
12
12
  from sdmxlib.model.dataflow import Dataflow
@@ -16,14 +16,16 @@ from statcatlib.model.agents import Agent
16
16
  from statcatlib.model.dataset import AggregateDataset
17
17
 
18
18
  if TYPE_CHECKING:
19
- from sdmxlib.api.registry import RestRegistry
19
+ from sdmxlib import ObsQuery
20
20
  from sdmxlib.local.registry import LocalRegistry
21
21
  from sdmxlib.model.in_memory_registry import InMemoryRegistry
22
+ from sdmxlib.model.urn import SdmxUrn
22
23
 
23
24
  from statcatlib.model.skos import ConceptRef
24
25
  from statcatlib.store.store import Catalog
25
26
 
26
27
  __all__ = [
28
+ "ObservationSource",
27
29
  "dataset_from_dataflow",
28
30
  "ingest_data",
29
31
  "ingest_sdmx",
@@ -34,6 +36,17 @@ __all__ = [
34
36
  _IDENTIFIER_RE = re.compile(r"^([^:]+):([^(]+)\(([^)]+)\)$")
35
37
 
36
38
 
39
+ class ObservationSource(Protocol):
40
+ """Where `ingest_data` reads observations from.
41
+
42
+ `sdmxlib.RestRegistry` is the usual one; `sdmxlib.LocalRegistry` fits too.
43
+ """
44
+
45
+ def obs(self, flow: SdmxUrn[Dataflow], /) -> ObsQuery:
46
+ """A lazy query over *flow*'s observations."""
47
+ ...
48
+
49
+
37
50
  def _resolve_latest(store: Catalog, identifier: str) -> str:
38
51
  """Resolve ``(latest)`` to the highest stored version for that agency:id."""
39
52
  if not identifier.endswith("(latest)"):
@@ -237,31 +250,40 @@ def _query_dimension_names(conn: object, dataflow_urn: str) -> list[Internationa
237
250
 
238
251
  def ingest_data(
239
252
  store: Catalog,
240
- reg: RestRegistry,
253
+ reg: ObservationSource,
241
254
  identifier: str,
242
255
  *,
243
256
  measure_dim: str | None = None,
244
257
  ) -> str:
245
258
  """Materialise observation data for one dataflow into the bundle's lake.
246
259
 
247
- Fetches the observations wide via ``reg.obs(...).pivot()``, writes them
248
- into ``lake.<agency>.df_<id>_<version>`` through the bundle's
249
- DuckLake-backed data store, and registers a
250
- `sdmxlib.DataflowBinding` so the dataflow is queryable through
251
- ``cat.observations(...)``.
260
+ Fetches the observations from *reg* in the long layout, one row per
261
+ observation with a column per dimension and measure, and writes them into
262
+ ``lake.<agency>.df_<id>_<version>`` through the bundle's DuckLake-backed
263
+ data store. A re-run replaces the table. It then registers a
264
+ `sdmxlib.DataflowBinding` for that table, so the dataflow reads back
265
+ through the co-located registry::
266
+
267
+ obs = cat.registry.obs(flow_urn)
268
+ obs.filter(obs.col("GEO") == "CA").frame()
269
+ obs.pivot(on=ds.measure_dim)
252
270
 
253
271
  Requires that ``ingest_sdmx`` has already been called for this
254
- identifier so the catalog record exists, and that ``store`` is a
255
- folder-mode `Catalog` (a DuckLake is ATTACHed on ``open()``).
272
+ identifier, so the catalog record and the DSD are stored, and that
273
+ ``store`` is a folder-mode `Catalog` (a DuckLake is ATTACHed on
274
+ ``open()``).
256
275
 
257
276
  Args:
258
277
  store: The catalog holding the `AggregateDataset` for *identifier*.
259
278
  Must be folder-mode — a DuckLake is ATTACHed on ``open()``.
260
- reg: REST registry to fetch observations from.
279
+ reg: Where to fetch observations from, usually a
280
+ `sdmxlib.RestRegistry`.
261
281
  identifier: Catalog identifier in ``AGENCY:ID(VERSION)`` form.
262
282
  ``(latest)`` resolves to the highest stored version.
263
- measure_dim: Dimension to pivot into measure columns. ``None``
264
- (default) pivots on the DSD's ``MeasureDimension``.
283
+ measure_dim: The dimension readers pivot on, recorded as
284
+ `AggregateDataset.measure_dim`. It does not change what is
285
+ stored. ``None`` (default) records none, and ``pivot()`` then
286
+ uses the DSD's ``MeasureDimension``.
265
287
 
266
288
  Returns:
267
289
  The fully-qualified table location (``<schema>.<table>``).
@@ -271,9 +293,12 @@ def ingest_data(
271
293
  ``ingest_sdmx`` first.
272
294
  ValueError: The dataset carries no ``sdmx_dataflow_urn``, so there is
273
295
  nothing to fetch observations for; *identifier* is not in
274
- ``AGENCY:ID(VERSION)`` form; or *measure_dim* is ``None`` and the
275
- DSD declares no ``MeasureDimension`` to pivot on.
296
+ ``AGENCY:ID(VERSION)`` form; the dataflow's DSD is not stored, so
297
+ no binding can describe the table; or *measure_dim* is not one of
298
+ the DSD's dimensions.
276
299
  """
300
+ import sdmxlib as sl # noqa: PLC0415
301
+
277
302
  identifier = _resolve_latest(store, identifier)
278
303
 
279
304
  ds = store.get(AggregateDataset, identifier)
@@ -290,31 +315,31 @@ def ingest_data(
290
315
  raise ValueError(msg)
291
316
  agency, flow_id, version = m.group(1), m.group(2), m.group(3)
292
317
 
293
- schema = agency.lower()
294
- table = f"df_{flow_id}_{version.replace('.', '_')}".lower()
295
-
296
- # Pull the wide dataframe and write through the bundle's lake. ``pivot()``
297
- # returns an eager DataFrame; with ``on=None`` it pivots on the DSD's
298
- # MeasureDimension, as ``reg.data(...).pivot()`` did.
299
318
  flow_urn = Dataflow.parse_urn(ds.sdmx_dataflow_urn)
300
- wide = reg.obs(flow_urn).pivot(on=measure_dim)
301
- store.data.write_observations(schema=schema, table=table, source=wide)
319
+ dsd = store.registry.dsd(flow_urn)
320
+ if dsd is None:
321
+ msg = f"The DSD of {ds.sdmx_dataflow_urn} is not stored — run ingest_sdmx with resolve=True first"
322
+ raise ValueError(msg)
323
+ dimensions = [d.id for d in dsd.dimensions]
324
+ if measure_dim is not None and measure_dim not in dimensions:
325
+ msg = f"measure_dim {measure_dim!r} is not a dimension of {dsd.id}: {', '.join(dimensions)}"
326
+ raise ValueError(msg)
302
327
 
303
- # Register a DataflowBinding so ``cat.observations(flow_urn)`` can find
304
- # the table. Derive identity column maps from the DSD when resolvable.
305
- import sdmxlib as sl # noqa: PLC0415
328
+ schema = agency.lower()
329
+ table = f"df_{flow_id}_{version.replace('.', '_')}".lower()
306
330
 
307
- flow = store.registry.get(flow_urn)
308
- if isinstance(flow, Dataflow) and isinstance(flow.structure, sl.Ref):
309
- dsd = store.registry.get(flow.structure.urn)
310
- if isinstance(dsd, DataStructure):
311
- binding = sl.DataflowBinding.from_dsd(
312
- dsd,
313
- dataflow=sl.Ref[Dataflow].unresolved(flow_urn),
314
- schema=schema,
315
- table=table,
316
- )
317
- store.registry.add(binding)
331
+ # Store the long layout. The binding below maps each DSD component to a
332
+ # column of the same name, which is the long layout; a pivoted table has a
333
+ # column per code instead and cannot be read through it (GH-29).
334
+ store.data.write_observations(schema=schema, table=table, source=reg.obs(flow_urn).frame())
335
+ store.registry.add(
336
+ sl.DataflowBinding.from_dsd(
337
+ dsd,
338
+ dataflow=sl.Ref[Dataflow].unresolved(flow_urn),
339
+ schema=schema,
340
+ table=table,
341
+ )
342
+ )
318
343
 
319
344
  ds.measure_dim = measure_dim
320
345
  store.put(ds)
@@ -933,18 +933,18 @@ class _LanguageIndex:
933
933
  ) -> list[Bm25HitRow]:
934
934
  """Run a BM25 query, returning raw result rows.
935
935
 
936
- The `# pyright: ignore[reportReturnType]` below is this module's only
937
- suppression, and is the shape sdmxlib's CLAUDE.md prescribes for a
938
- gradual-typing boundary: the one unsafe step, kept to the accessor
939
- that performs it. `to_list()` is declared `List[dict[Unknown, Unknown]]`
936
+ The `# pyrefly: ignore[bad-return]` below (and its twin in `vector`)
937
+ is this module's only suppression, and is the shape sdmxlib's
938
+ CLAUDE.md prescribes for a gradual-typing boundary: the one unsafe
939
+ step, kept to the accessor that performs it. `to_list()` is declared `List[dict[Unknown, Unknown]]`
940
940
  — LanceDB hands back rows of the schema *this module declared* and
941
941
  says nothing about their shape. Validating 19 keys per row on every
942
942
  search to recover what `make_schema` already guarantees would cost
943
943
  more than it proves.
944
944
 
945
- Not `# type: ignore`: that form is unattributable, so neither checker
946
- can tell a dead comment from one carrying the other's error, and
947
- `reportUnnecessaryTypeIgnoreComment` stops being trustworthy.
945
+ Not `# type: ignore`: that form names no checker and no error kind,
946
+ so a dead one cannot be told from a live one, and pyrefly's
947
+ `unused-ignore` stops being trustworthy.
948
948
  """
949
949
  if self._tbl is None:
950
950
  return []
@@ -957,7 +957,7 @@ class _LanguageIndex:
957
957
  where = self._where(*filters)
958
958
  if where:
959
959
  search = search.where(where, prefilter=True)
960
- return search.limit(limit).select(select_cols).to_list() # pyright: ignore[reportReturnType] # pyrefly: ignore[bad-return]
960
+ return search.limit(limit).select(select_cols).to_list() # pyrefly: ignore[bad-return]
961
961
  except Exception: # noqa: BLE001
962
962
  return []
963
963
 
@@ -978,7 +978,7 @@ class _LanguageIndex:
978
978
  where = self._where(*filters)
979
979
  if where:
980
980
  search = search.where(where, prefilter=True)
981
- return search.limit(limit).select(select_cols).to_list() # pyright: ignore[reportReturnType] # pyrefly: ignore[bad-return]
981
+ return search.limit(limit).select(select_cols).to_list() # pyrefly: ignore[bad-return]
982
982
  except Exception: # noqa: BLE001
983
983
  return []
984
984
 
@@ -4,7 +4,7 @@ Scopes represent positions in the catalog graph. Navigation builds new
4
4
  scopes without executing SQL; only terminal methods (``.collect()``,
5
5
  ``.count()``) hit the database.
6
6
 
7
- Usage::
7
+ Usage:
8
8
 
9
9
  store.themes["https://example.com/themes"].collect() # ItemList[ConceptRef]
10
10
  store.themes["https://example.com/themes"]["13"].collect() # Concept
@@ -143,9 +143,10 @@ class _SdmxEdge(TypedDict):
143
143
  #:
144
144
  #: Discriminated on an explicit ``kind`` rather than on key presence. Testing
145
145
  #: ``"custom_count_sql" in edge`` to conclude ``custom_exists_sql`` also exists
146
- #: is sound given the TypedDicts, but only basedpyright narrows it; pyrefly
147
- #: reported reading the second key as an error. Two checkers disagreeing about
148
- #: whether a branch is safe is reason enough to say which branch it is.
146
+ #: is sound given the TypedDicts, but pyrefly reports reading the second key as
147
+ #: an error, where basedpyright (run beside it until GH-7) narrowed it. Two
148
+ #: checkers disagreeing about whether a branch is safe is reason enough to say
149
+ #: which branch it is.
149
150
  type _Edge = _BridgeEdge | _SdmxEdge
150
151
 
151
152
 
@@ -1131,8 +1132,16 @@ class DatasetScope:
1131
1132
 
1132
1133
  Supported edges: ``"codelist"``.
1133
1134
 
1134
- Example::
1135
+ Args:
1136
+ edge: The kind of entity the datasets share.
1137
+
1138
+ Returns:
1139
+ A query over the datasets linked to this one through *edge*.
1135
1140
 
1141
+ Raises:
1142
+ ValueError: *edge* is not a supported edge type.
1143
+
1144
+ Example:
1136
1145
  store.dataset("id").related("codelist").include("publisher").limit(10).collect()
1137
1146
  """
1138
1147
  from statcatlib.model.dataset import BaseDataset # noqa: PLC0415
@@ -1163,8 +1172,14 @@ class DatasetScope:
1163
1172
 
1164
1173
  Gremlin equivalent: ``g.V(id).repeat(in('derivedFrom')).times(depth)``
1165
1174
 
1166
- Example::
1175
+ Args:
1176
+ relationship: Follow only lineage edges of this relationship; all of them when ``None``.
1177
+ depth: How many lineage hops to follow.
1178
+
1179
+ Returns:
1180
+ A query over this dataset's ancestors, to *depth* hops.
1167
1181
 
1182
+ Example:
1168
1183
  store.dataset("id").upstream(depth=3).include("publisher").collect()
1169
1184
  """
1170
1185
  from statcatlib.model.dataset import BaseDataset # noqa: PLC0415
@@ -1207,8 +1222,14 @@ class DatasetScope:
1207
1222
 
1208
1223
  Gremlin equivalent: ``g.V(id).repeat(out('derivedFrom')).times(depth)``
1209
1224
 
1210
- Example::
1225
+ Args:
1226
+ relationship: Follow only lineage edges of this relationship; all of them when ``None``.
1227
+ depth: How many lineage hops to follow.
1228
+
1229
+ Returns:
1230
+ A query over this dataset's descendants, to *depth* hops.
1211
1231
 
1232
+ Example:
1212
1233
  store.dataset("id").downstream().include("publisher").collect()
1213
1234
  """
1214
1235
  from statcatlib.model.dataset import BaseDataset # noqa: PLC0415
@@ -18,8 +18,10 @@ _cache: dict[str, _UnloadedType] = {}
18
18
  class NotLoadedError(AttributeError):
19
19
  """Raised when accessing a relationship that was not loaded in this query.
20
20
 
21
- Example::
21
+ Args:
22
+ field: The relationship that was not loaded, as `include` names it.
22
23
 
24
+ Example:
23
25
  ds = store.query(BaseDataset).include("publisher").first()
24
26
  ds.themes # NotLoadedError: 'themes' was not loaded.
25
27
  # Add 'themes' to .include() or call .full().
@@ -86,7 +88,7 @@ def make_unloaded(field: str) -> Any:
86
88
 
87
89
  (No `# noqa: ANN401` yet — ANN401 is still ignored project-wide until
88
90
  #109 step 3, and a suppression that suppresses nothing is the thing
89
- `reportUnnecessaryTypeIgnoreComment` exists to catch. Add it with the
91
+ pyrefly's `unused-ignore` exists to catch. Add it with the
90
92
  rule.)
91
93
  """
92
94
  if field not in _cache:
@@ -36,7 +36,7 @@ from sdmxlib import InternationalString
36
36
  from sdmxlib.model.registry import artefact_urn
37
37
  from sdmxlib.storage.schema import SCHEMA_VERSION as _SDMX_SCHEMA_VERSION
38
38
 
39
- from statcatlib.bundle import Embedded
39
+ from statcatlib.bundle import Embedded, normalize_lake_data_path, read_current_data_path
40
40
  from statcatlib.model.agents import Agent, ContactPoint
41
41
  from statcatlib.model.classification import Standard
42
42
  from statcatlib.model.dataset import (
@@ -543,9 +543,10 @@ class _EntitySpec[R: NamedTuple, E]:
543
543
 
544
544
  Bounding ``R`` by `NamedTuple` is what makes `columns` derivable —
545
545
  it is the one spelling of "a row type" that basedpyright *and* pyrefly
546
- both accept. A `Protocol` carrying ``_fields`` does not: declared
547
- `ClassVar` (which it truly is) basedpyright rejects ``type[R]``;
548
- declared as an instance attribute pyrefly rejects the specialization.
546
+ both accepted while both ran (until GH-7). A `Protocol` carrying
547
+ ``_fields`` did not: declared `ClassVar` (which it truly is) basedpyright
548
+ rejected ``type[R]``; declared as an instance attribute pyrefly rejects
549
+ the specialization.
549
550
 
550
551
  Attributes:
551
552
  table: The DuckDB table name, unqualified.
@@ -590,8 +591,8 @@ class EmbeddingModel(Protocol):
590
591
  ``inputs`` (older versions: ``sentences``), and declares a return union
591
592
  wide enough to include ``dict[str, Tensor]``. Promising
592
593
  ``-> Iterable[float]`` here was the first attempt and no real encoder
593
- satisfied it; worse, narrowing to it with `isinstance` is unsound, and
594
- basedpyright says so — a runtime_checkable Protocol checks method
594
+ satisfied it; worse, narrowing to it with `isinstance` is unsound, as
595
+ basedpyright reported — a runtime_checkable Protocol checks method
595
596
  *presence*, never signatures.
596
597
 
597
598
  So: two overloads, one per shape statcatlib passes, `object` out, and
@@ -1157,7 +1158,7 @@ class _SortSpec:
1157
1158
  #:
1158
1159
  #: A union rather than `Hit[object]` because attrs generics are invariant:
1159
1160
  #: `Hit` is frozen and `SearchResult.items` could be a `Sequence`, and
1160
- #: basedpyright still reports the parameter invariant, since the synthesised
1161
+ #: basedpyright reported the parameter invariant, since the synthesised
1161
1162
  #: `__init__` puts `T` in a parameter position. `Hit[CodeHit]` is therefore not
1162
1163
  #: a `Hit[object]`, and no amount of freezing changes that.
1163
1164
  type AnyHit = Hit[BaseDataset] | Hit[CodeHit] | Hit[ConceptHit] | Hit[ReferenceMetadataHit]
@@ -2847,8 +2848,8 @@ class Catalog:
2847
2848
  # abfss:// URI) via ``data_path``.
2848
2849
  if self._bundle_dir is not None:
2849
2850
  attach_url = f"'ducklake:{self._bundle_dir / 'lake.ducklake'}'"
2850
- data_path_literal = self._resolve_data_path(self._bundle_dir)
2851
- attach_options = self._attach_options(data_path_literal)
2851
+ data_path_literal, override = self._resolve_data_path(self._bundle_dir)
2852
+ attach_options = self._attach_options(data_path_literal, override=override)
2852
2853
 
2853
2854
  self._data = sdmxlib.DuckDBDataStore.from_registry(
2854
2855
  self._registry,
@@ -2914,7 +2915,11 @@ class Catalog:
2914
2915
  msg = "data_path is not supported with an in-memory catalog (':memory:')"
2915
2916
  raise ValueError(msg)
2916
2917
  return duckdb.connect(":memory:"), None
2917
- path = Path(self._uri)
2918
+ # Absolute, so the stamp DuckLake records on first open is too: a
2919
+ # relative one is resolved against whatever the working directory is
2920
+ # when the lake is read (GH-23). `absolute()`, not `resolve()`, which
2921
+ # follows symlinks and would respell a catalog created by absolute path.
2922
+ path = Path(self._uri).absolute()
2918
2923
  if self._read_only:
2919
2924
  meta_path = path / "meta.duckdb"
2920
2925
  if not meta_path.exists():
@@ -2929,7 +2934,7 @@ class Catalog:
2929
2934
  path.mkdir(parents=True, exist_ok=True)
2930
2935
  return duckdb.connect(str(path / "meta.duckdb")), path
2931
2936
 
2932
- def _attach_options(self, data_path_literal: str | None) -> str:
2937
+ def _attach_options(self, data_path_literal: str | None, *, override: bool = False) -> str:
2933
2938
  """Render the DuckLake ATTACH option list.
2934
2939
 
2935
2940
  A ``None`` literal means honour the lake's own stamp: DuckLake only
@@ -2944,31 +2949,73 @@ class Catalog:
2944
2949
  allowed under ``READ_ONLY``. The default (``data_path=None``) stays
2945
2950
  a checked ``<bundle>/parquet/``, so a plain ``Catalog(path)`` on a
2946
2951
  split bundle still fails at ATTACH rather than reading an empty
2947
- local directory.
2952
+ local directory. *override* is the one exception, for a read-only
2953
+ open of a catalog stamped with a relative path (`_resolve_data_path`).
2948
2954
  """
2949
2955
  opts = [] if data_path_literal is None else [f"DATA_PATH '{data_path_literal}'"]
2950
- if data_path_literal is not None and self._data_path is not None:
2956
+ if data_path_literal is not None and (self._data_path is not None or override):
2951
2957
  opts.append("OVERRIDE_DATA_PATH true")
2952
2958
  if self._read_only:
2953
2959
  opts.append("READ_ONLY")
2954
2960
  return f"({', '.join(opts)})" if opts else ""
2955
2961
 
2956
- def _resolve_data_path(self, bundle_dir: Path) -> str | None:
2957
- """Return the DuckLake DATA_PATH literal, with trailing slash.
2962
+ def _resolve_data_path(self, bundle_dir: Path) -> tuple[str | None, bool]:
2963
+ """Return the DuckLake DATA_PATH literal, with trailing slash, and whether to override.
2958
2964
 
2959
2965
  Remote URIs (``abfss://``, ``s3://``, ``gs://``, etc.) are passed
2960
2966
  through verbatim; local paths are rendered as ``<bundle>/parquet/``
2961
- when no override is set.
2967
+ when no override is set, reconciled with the lake's stamp by
2968
+ `_colocated_data_path`.
2962
2969
 
2963
2970
  Returns `None` for `statcatlib.bundle.Embedded` — meaning "omit the
2964
2971
  option entirely and let the lake's own stamp stand".
2965
2972
  """
2966
2973
  if isinstance(self._data_path, Embedded):
2967
- return None
2974
+ return None, False
2968
2975
  if self._data_path is None:
2969
- return f"{bundle_dir / 'parquet'}/"
2976
+ return self._colocated_data_path(bundle_dir)
2970
2977
  raw = str(self._data_path)
2971
- return raw if raw.endswith("/") else f"{raw}/"
2978
+ return (raw if raw.endswith("/") else f"{raw}/"), False
2979
+
2980
+ def _colocated_data_path(self, bundle_dir: Path) -> tuple[str, bool]:
2981
+ """The DATA_PATH for a default open: ``<bundle>/parquet/``, as the lake spells it.
2982
+
2983
+ DuckLake compares DATA_PATH with its stamp as strings, so the same
2984
+ directory spelled differently was refused (GH-23). The stamp is read
2985
+ first:
2986
+
2987
+ - the same directory, spelled any way (a relative open, a symlink):
2988
+ passed back verbatim, which DuckLake accepts unchanged;
2989
+ - relative and ending in ``<bundle name>/parquet``, as
2990
+ ``Catalog("relative")`` stamped it before GH-23: DuckLake resolves
2991
+ it against the working directory, so it is replaced by the absolute
2992
+ path. A read-write open rewrites the stamp, which repairs the
2993
+ catalog; a read-only one overrides it for this attach only;
2994
+ - anything else, a remote URI or another directory: the absolute
2995
+ path, unchanged, so a split bundle still fails at ATTACH rather than
2996
+ reading an empty local directory.
2997
+ """
2998
+ parquet = bundle_dir / "parquet"
2999
+ target = f"{parquet}/"
3000
+ lake = bundle_dir / "lake.ducklake"
3001
+ if not lake.exists():
3002
+ return target, False
3003
+ try:
3004
+ stamp = read_current_data_path(lake)
3005
+ except duckdb.Error, FileNotFoundError:
3006
+ return target, False
3007
+ stamped = Path(stamp)
3008
+ if "://" in stamp or stamp == target:
3009
+ return target, False
3010
+ if stamped.is_absolute():
3011
+ return (stamp, False) if stamped.resolve() == parquet.resolve() else (target, False)
3012
+ colocated = stamped.name == "parquet" and stamped.parent.name in {"", bundle_dir.name}
3013
+ if not colocated:
3014
+ return target, False
3015
+ if self._read_only:
3016
+ return target, True
3017
+ normalize_lake_data_path(lake, target)
3018
+ return target, False
2972
3019
 
2973
3020
  @classmethod
2974
3021
  def from_connection(
@@ -3228,8 +3275,10 @@ class Catalog:
3228
3275
  the caller's responsibility — the encoder name is opaque to
3229
3276
  statcatlib.
3230
3277
 
3231
- Example::
3278
+ Args:
3279
+ model: The encoder, the one `embedding_model_name` names.
3232
3280
 
3281
+ Example:
3233
3282
  store = Catalog("data/catalog").open()
3234
3283
  if store.embedding_model_name:
3235
3284
  from sentence_transformers import SentenceTransformer
@@ -6391,7 +6440,7 @@ class Catalog:
6391
6440
  # arg 2 must be a type`. Found by trying it; five tests failed.
6392
6441
  # Entity queries never see the alias, which is why the same fix holds
6393
6442
  # there.
6394
- items: list[T] = [ # pyright: ignore[reportAssignmentType] # pyrefly: ignore[bad-assignment]
6443
+ items: list[T] = [ # pyrefly: ignore[bad-assignment]
6395
6444
  ds_by_id[ident] for ident in qr.rows if ident in ds_by_id
6396
6445
  ]
6397
6446
 
@@ -6447,7 +6496,7 @@ class Catalog:
6447
6496
  # returns `Any` here, and the `isinstance` is what soundly recovers `T`
6448
6497
  # — the bridge back from runtime dispatch to the declared return type,
6449
6498
  # not a redundant assertion. It is also why this path needs no
6450
- # `pyright: ignore` while the dataset path above does.
6499
+ # suppression while the dataset path above does.
6451
6500
  items: list[T] = []
6452
6501
  for raw in rows:
6453
6502
  item = spec.build(raw)
@@ -48,8 +48,15 @@ def _build(package: str, artefact_class: str, agency: str, artefact_id: str, ver
48
48
  def codelist(agency: str, id: str, version: str = "1.0") -> str: # noqa: A002
49
49
  """Build a codelist URN string.
50
50
 
51
- Example::
51
+ Args:
52
+ agency: The maintenance agency's id.
53
+ id: The codelist's id.
54
+ version: Its version.
52
55
 
56
+ Returns:
57
+ The URN, as SDMX spells it.
58
+
59
+ Example:
53
60
  sc.urn.codelist("SDMX", "CL_GEO", "1.0")
54
61
  # → "urn:sdmx:org.sdmx.infomodel.codelist.Codelist=SDMX:CL_GEO(1.0)"
55
62
  """
@@ -59,8 +66,15 @@ def codelist(agency: str, id: str, version: str = "1.0") -> str: # noqa: A002
59
66
  def dataflow(agency: str, id: str, version: str = "1.0") -> str: # noqa: A002
60
67
  """Build a dataflow URN string.
61
68
 
62
- Example::
69
+ Args:
70
+ agency: The maintenance agency's id.
71
+ id: The dataflow's id.
72
+ version: Its version.
73
+
74
+ Returns:
75
+ The URN, as SDMX spells it.
63
76
 
77
+ Example:
64
78
  sc.urn.dataflow("ESTAT", "NAMA_10_GDP", "1.0")
65
79
  # → "urn:sdmx:org.sdmx.infomodel.datastructure.Dataflow=ESTAT:NAMA_10_GDP(1.0)"
66
80
  """
@@ -70,8 +84,15 @@ def dataflow(agency: str, id: str, version: str = "1.0") -> str: # noqa: A002
70
84
  def data_structure(agency: str, id: str, version: str = "1.0") -> str: # noqa: A002
71
85
  """Build a data structure definition URN string.
72
86
 
73
- Example::
87
+ Args:
88
+ agency: The maintenance agency's id.
89
+ id: The data structure definition's id.
90
+ version: Its version.
91
+
92
+ Returns:
93
+ The URN, as SDMX spells it.
74
94
 
95
+ Example:
75
96
  sc.urn.data_structure("ESTAT", "DSD_NAMA_10_GDP", "1.0")
76
97
  # → "urn:sdmx:org.sdmx.infomodel.datastructure.DataStructure=ESTAT:DSD_NAMA_10_GDP(1.0)"
77
98
  """
@@ -81,8 +102,15 @@ def data_structure(agency: str, id: str, version: str = "1.0") -> str: # noqa:
81
102
  def concept_scheme(agency: str, id: str, version: str = "1.0") -> str: # noqa: A002
82
103
  """Build a concept scheme URN string.
83
104
 
84
- Example::
105
+ Args:
106
+ agency: The maintenance agency's id.
107
+ id: The concept scheme's id.
108
+ version: Its version.
85
109
 
110
+ Returns:
111
+ The URN, as SDMX spells it.
112
+
113
+ Example:
86
114
  sc.urn.concept_scheme("SDMX", "CS_FREQ", "1.0")
87
115
  # → "urn:sdmx:org.sdmx.infomodel.conceptscheme.ConceptScheme=SDMX:CS_FREQ(1.0)"
88
116
  """
@@ -92,17 +120,18 @@ def concept_scheme(agency: str, id: str, version: str = "1.0") -> str: # noqa:
92
120
  def parse(urn: str) -> SdmxUrn[object]:
93
121
  """Parse a URN string into its components.
94
122
 
95
- Returns an ``SdmxUrn`` object with ``.agency``, ``.id``, ``.version``,
96
- ``.package``, and ``.artefact_class`` attributes.
123
+ Args:
124
+ urn: An SDMX URN string. One that is not valid raises `ValueError`.
97
125
 
98
- Example::
126
+ Returns:
127
+ The parsed URN, with ``.agency``, ``.id``, ``.version``, ``.package``
128
+ and ``.artefact_class``.
99
129
 
130
+ Example:
100
131
  parsed = sc.urn.parse("urn:sdmx:org.sdmx.infomodel.codelist.Codelist=SDMX:CL_GEO(1.0)")
101
132
  parsed.agency # "SDMX"
102
133
  parsed.id # "CL_GEO"
103
134
  parsed.version # "1.0"
104
135
  str(parsed) # round-trips to original URN string
105
-
106
- Raises ``ValueError`` if the string is not a valid SDMX URN.
107
136
  """
108
137
  return SdmxUrn.parse(urn)
File without changes