statcatlib 0.8.0__tar.gz → 0.9.1__tar.gz

{statcatlib-0.8.0 → statcatlib-0.9.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: statcatlib
-Version: 0.8.0
+Version: 0.9.1
 Summary: Queryable open data catalog engine for DCAT-AP, StatDCAT-AP, CKAN, and SDMX
 Keywords: sdmx,dcat,open-data,catalog,duckdb,statistical-data,ckan
 Author: gabrielgellner
@@ -11,19 +11,19 @@ Classifier: Development Status :: 3 - Alpha
 Classifier: Intended Audience :: Developers
 Classifier: Intended Audience :: Science/Research
 Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
 Classifier: Topic :: Database
 Classifier: Topic :: Scientific/Engineering :: Information Analysis
 Classifier: Topic :: Internet :: WWW/HTTP :: Indexing/Search
 Requires-Dist: attrs>=26.1.0
-Requires-Dist: duckdb>=1.1,<2
+Requires-Dist: duckdb>=2.0.0
 Requires-Dist: httpx>=0.28
-Requires-Dist: sdmxlib>=0.67
+Requires-Dist: sdmxlib>=0.70.1
 Requires-Dist: lancedb>=0.39 ; extra == 'search'
 Requires-Dist: sentence-transformers>=3.0 ; extra == 'search'
 Requires-Dist: typer>=0.15 ; extra == 'search'
 Requires-Dist: rich>=15.0 ; extra == 'search'
-Requires-Python: >=3.13
+Requires-Python: >=3.14
 Project-URL: Homepage, https://github.com/StatCat-Suite/statcatlib
 Project-URL: Repository, https://github.com/StatCat-Suite/statcatlib
 Project-URL: Documentation, https://statcat-suite.github.io/statcatlib
@@ -65,7 +65,7 @@ SQL JOINs are the graph traversal. No network round-trips at query time.
 
 ## Installation
 
-Requires Python 3.13+. Managed with [uv](https://docs.astral.sh/uv/).
+Requires Python 3.14+. Managed with [uv](https://docs.astral.sh/uv/).
 
 ```bash
 uv add statcatlib

{statcatlib-0.8.0 → statcatlib-0.9.1}/README.md

@@ -31,7 +31,7 @@ SQL JOINs are the graph traversal. No network round-trips at query time.
 
 ## Installation
 
-Requires Python 3.13+. Managed with [uv](https://docs.astral.sh/uv/).
+Requires Python 3.14+. Managed with [uv](https://docs.astral.sh/uv/).
 
 ```bash
 uv add statcatlib

{statcatlib-0.8.0 → statcatlib-0.9.1}/pyproject.toml

@@ -1,11 +1,11 @@
 [project]
 name = "statcatlib"
-version = "0.8.0"
+version = "0.9.1"
 description = "Queryable open data catalog engine for DCAT-AP, StatDCAT-AP, CKAN, and SDMX"
 license = "Apache-2.0"
 license-files = ["LICENSE"]
 readme = "README.md"
-requires-python = ">=3.13"
+requires-python = ">=3.14"
 keywords = [
     "sdmx",
     "dcat",
@@ -20,16 +20,16 @@ classifiers = [
     "Intended Audience :: Developers",
     "Intended Audience :: Science/Research",
     "Programming Language :: Python :: 3",
-    "Programming Language :: Python :: 3.13",
+    "Programming Language :: Python :: 3.14",
     "Topic :: Database",
     "Topic :: Scientific/Engineering :: Information Analysis",
     "Topic :: Internet :: WWW/HTTP :: Indexing/Search",
 ]
 dependencies = [
     "attrs>=26.1.0",
-    "duckdb>=1.1,<2",
+    "duckdb>=2.0.0",
     "httpx>=0.28",
-    "sdmxlib>=0.67",
+    "sdmxlib>=0.70.1",
 ]
 
 [[project.authors]]
@@ -79,7 +79,7 @@ markers = [
 ]
 
 [tool.basedpyright]
-pythonVersion = "3.13"
+pythonVersion = "3.14"
 typeCheckingMode = "standard"
 venvPath = "."
 venv = ".venv"

{statcatlib-0.8.0 → statcatlib-0.9.1}/pyproject.toml.orig

@@ -1,6 +1,6 @@
 [project]
 name = "statcatlib"
-version = "0.8.0"
+version = "0.9.1"
 description = "Queryable open data catalog engine for DCAT-AP, StatDCAT-AP, CKAN, and SDMX"
 license = "Apache-2.0"
 license-files = ["LICENSE"]
@@ -8,27 +8,30 @@ readme = "README.md"
 authors = [
     { name = "gabrielgellner", email = "gabrielgellner@gmail.com" }
 ]
-requires-python = ">=3.13"
+requires-python = ">=3.14"
 keywords = ["sdmx", "dcat", "open-data", "catalog", "duckdb", "statistical-data", "ckan"]
 classifiers = [
     "Development Status :: 3 - Alpha",
     "Intended Audience :: Developers",
     "Intended Audience :: Science/Research",
     "Programming Language :: Python :: 3",
-    "Programming Language :: Python :: 3.13",
+    "Programming Language :: Python :: 3.14",
     "Topic :: Database",
     "Topic :: Scientific/Engineering :: Information Analysis",
     "Topic :: Internet :: WWW/HTTP :: Indexing/Search",
 ]
 dependencies = [
     "attrs>=26.1.0",
-    # Below 2: on Python 3.13 the newest sdmxlib that resolves is 0.67.1 (0.68+
-    # needs 3.14), and it does not cap DuckDB itself, but its lake writes fail
-    # under DuckDB 2.0 — sdmxlib 0.69.1 added the same cap for the same reason.
-    # Lift it with the move to an sdmxlib that supports DuckDB 2.
-    "duckdb>=1.1,<2",
+    # >=2.0: sdmxlib 0.70 moved its store to DuckDB 2.0, and the catalog
+    # shares that store's connection. A catalog written here cannot be opened
+    # by DuckDB 1.x, so by statcatlib 0.8 or earlier.
+    "duckdb>=2.0.0",
     "httpx>=0.28",
-    "sdmxlib>=0.67",
+    # >=0.70: needs Python 3.14 and DuckDB 2.0; `reg.data()` is gone, and a
+    # component without a concept is refused as it is built. >=0.70.1:
+    # `reg.obs()` asks Eurostat for CSV it serves, so `ingest_data()` can
+    # fetch from it (sdmxlib GH-50).
+    "sdmxlib>=0.70.1",
 ]
 
 [project.optional-dependencies]
@@ -95,7 +98,7 @@ dev = [
 ]
 
 [tool.basedpyright]
-pythonVersion = "3.13"
+pythonVersion = "3.14"
 typeCheckingMode = "standard"
 venvPath = "."
 venv = ".venv"

{statcatlib-0.8.0 → statcatlib-0.9.1}/src/statcatlib/__init__.py

@@ -83,6 +83,7 @@ from statcatlib.store import (
     Hit,
     SEARCH_CONTEXT_ANNOTATION_TYPE,
     SEARCH_EXCLUDE_ANNOTATION_TYPE,
+    IncompatibleCatalogError,
     IncompatibleIndexError,
     QueryBuilder,
     ReferenceMetadataBlend,
@@ -145,6 +146,7 @@ __all__ = [
     "Frequency",
     "GeospatialDataset",
     "Hit",
+    "IncompatibleCatalogError",
     "IncompatibleIndexError",
     "InternationalString",
     "LicenceDocument",

{statcatlib-0.8.0 → statcatlib-0.9.1}/src/statcatlib/sources/ckan.py

@@ -206,7 +206,7 @@ def _parse_istring(
                 if fr_val2 := parsed.get(fr_key):
                     kw2["fr"] = fr_val2
                 return InternationalString(**kw2)
-        except (json.JSONDecodeError, ValueError):
+        except json.JSONDecodeError, ValueError:
             pass
         return InternationalString(en=raw or fallback_en)
     return InternationalString(en=fallback_en)
@@ -231,7 +231,7 @@ def _parse_keywords(raw: object) -> list[InternationalString]:
     if isinstance(raw, str):
         try:
             raw = json.loads(raw)
-        except (json.JSONDecodeError, ValueError):
+        except json.JSONDecodeError, ValueError:
             return []
     if not isinstance(raw, dict):
         return []
@@ -255,7 +255,7 @@ def _parse_frequency(raw: object) -> Frequency | None:
             parsed = json.loads(raw)
             if isinstance(parsed, dict):
                 raw = parsed
-        except (json.JSONDecodeError, ValueError):
+        except json.JSONDecodeError, ValueError:
             pass
     if isinstance(raw, dict):
         en_label = (raw.get("en") or "").strip()
@@ -285,7 +285,7 @@ def _parse_subjects(raw: object) -> list[ConceptRef]:
     if isinstance(raw, str):
         try:
             raw = json.loads(raw)
-        except (json.JSONDecodeError, ValueError):
+        except json.JSONDecodeError, ValueError:
             raw = [raw]
     if not isinstance(raw, list):
         return []
@@ -302,7 +302,7 @@ def _parse_spatial(raw: object) -> list[SpatialCoverage]:
     if isinstance(raw, str):
         try:
             raw = json.loads(raw)
-        except (json.JSONDecodeError, ValueError):
+        except json.JSONDecodeError, ValueError:
             raw = [raw]
     if not isinstance(raw, list):
         return []
@@ -427,7 +427,7 @@ def _ckan_package_to_dataset(raw: dict[str, object]) -> OpenDataset:
     if isinstance(lang_raw, str):
         try:
             lang_raw = json.loads(lang_raw)
-        except (json.JSONDecodeError, ValueError):
+        except json.JSONDecodeError, ValueError:
             lang_raw = [lang_raw]
     language: list[str] = [str(lang) for lang in _seq(lang_raw) if lang]
 

{statcatlib-0.8.0 → statcatlib-0.9.1}/src/statcatlib/sources/sdmx.py

@@ -240,8 +240,8 @@ def ingest_data(
 ) -> str:
     """Materialise observation data for one dataflow into the bundle's lake.
 
-    Fetches the (wide) observations via ``reg.data(...).pivot()``, writes
-    them into ``lake.<agency>.df_<id>_<version>`` through the bundle's
+    Fetches the observations wide via ``reg.obs(...).pivot()``, writes them
+    into ``lake.<agency>.df_<id>_<version>`` through the bundle's
     DuckLake-backed data store, and registers a
     `sdmxlib.DataflowBinding` so the dataflow is queryable through
     ``cat.observations(...)``.
@@ -257,7 +257,7 @@ def ingest_data(
         identifier: Catalog identifier in ``AGENCY:ID(VERSION)`` form.
             ``(latest)`` resolves to the highest stored version.
         measure_dim: Dimension to pivot into measure columns. ``None``
-            (default) leaves the data long.
+            (default) pivots on the DSD's ``MeasureDimension``.
 
     Returns:
         The fully-qualified table location (``<schema>.<table>``).
@@ -266,8 +266,9 @@ def ingest_data(
         KeyError: No `AggregateDataset` is stored under *identifier* — run
             ``ingest_sdmx`` first.
         ValueError: The dataset carries no ``sdmx_dataflow_urn``, so there is
-            nothing to fetch observations for; or *identifier* is not in
-            ``AGENCY:ID(VERSION)`` form.
+            nothing to fetch observations for; *identifier* is not in
+            ``AGENCY:ID(VERSION)`` form; or *measure_dim* is ``None`` and the
+            DSD declares no ``MeasureDimension`` to pivot on.
     """
     identifier = _resolve_latest(store, identifier)
 
@@ -289,21 +290,16 @@ def ingest_data(
     table = f"df_{flow_id}_{version.replace('.', '_')}".lower()
 
     # Pull the wide dataframe and write through the bundle's lake. ``pivot()``
-    # returns a LazyFrame; the data store collects it via Arrow under the hood.
-    lf = reg.data(
-        Dataflow,
-        agency=agency,
-        id=flow_id,
-        version=version,
-        measure_dim=measure_dim,
-    ).pivot()
-    store.data.write_observations(schema=schema, table=table, source=lf)
+    # returns an eager DataFrame; with ``on=None`` it pivots on the DSD's
+    # MeasureDimension, as ``reg.data(...).pivot()`` did.
+    flow_urn = Dataflow.parse_urn(ds.sdmx_dataflow_urn)
+    wide = reg.obs(flow_urn).pivot(on=measure_dim)
+    store.data.write_observations(schema=schema, table=table, source=wide)
 
     # Register a DataflowBinding so ``cat.observations(flow_urn)`` can find
     # the table. Derive identity column maps from the DSD when resolvable.
     import sdmxlib as sl  # noqa: PLC0415
 
-    flow_urn = Dataflow.parse_urn(ds.sdmx_dataflow_urn)
     flow = store.registry.get(flow_urn)
     if isinstance(flow, Dataflow) and isinstance(flow.structure, sl.Ref):
         dsd = store.registry.get(flow.structure.urn)

{statcatlib-0.8.0 → statcatlib-0.9.1}/src/statcatlib/store/__init__.py

@@ -28,6 +28,7 @@ from statcatlib.store.store import (
     Hit,
     SEARCH_CONTEXT_ANNOTATION_TYPE,
     SEARCH_EXCLUDE_ANNOTATION_TYPE,
+    IncompatibleCatalogError,
     IncompatibleIndexError,
     QueryBuilder,
     ReferenceMetadataBlend,

{statcatlib-0.8.0 → statcatlib-0.9.1}/src/statcatlib/store/lance.py

@@ -1043,7 +1043,7 @@ class LanceIndex:
         """
         try:
             data = json.loads((self._path / _MANIFEST_FILE).read_text())
-        except (OSError, ValueError):
+        except OSError, ValueError:
             return None
         return data if isinstance(data, dict) else None
 

{statcatlib-0.8.0 → statcatlib-0.9.1}/src/statcatlib/store/store.py

@@ -27,6 +27,7 @@ import sdmxlib.model.datastructure
 from attrs import Factory, define, field
 from sdmxlib import InternationalString
 from sdmxlib.model.registry import artefact_urn
+from sdmxlib.storage.schema import SCHEMA_VERSION as _SDMX_SCHEMA_VERSION
 
 from statcatlib.bundle import Embedded
 from statcatlib.model.agents import Agent, ContactPoint
@@ -143,6 +144,60 @@ class IncompatibleIndexError(ValueError):
     """
 
 
+class IncompatibleCatalogError(RuntimeError):
+    """Raised on open when the catalog's ``sdmx`` schema is another sdmxlib's.
+
+    sdmxlib treats its store as a cache: on a schema-version mismatch,
+    `LocalRegistry` drops the ``sdmx`` schema and rebuilds it empty. Here
+    that schema is not a cache. It is the structural half of the catalog,
+    which ``statcat.dataset`` links into by URN, so the rebuild would leave
+    every dataset pointing at nothing, and structural queries answering
+    empty rather than failing.
+    """
+
+
+def _refuse_sdmx_schema_rebuild(conn: duckdb.DuckDBPyConnection) -> None:
+    """Refuse to open a catalog whose SDMX structure sdmxlib would drop.
+
+    Runs before `LocalRegistry.from_connection`, which is where the drop
+    happens. A missing ``sdmx`` schema, or one at another version that holds
+    no artefacts, has nothing to lose, so sdmxlib is left to create or
+    rebuild it.
+
+    Read-only opens are refused here too. sdmxlib would refuse them itself,
+    but its message says to reopen read-write and rebuild in place, which
+    for a catalog is the data loss this guard exists to prevent.
+
+    Args:
+        conn: The catalog's connection, before sdmxlib has touched it.
+
+    Raises:
+        IncompatibleCatalogError: The stored ``sdmx`` schema is at another
+            version and holds artefacts.
+    """
+    try:
+        row = conn.execute("""
+            SELECT value FROM sdmx._meta WHERE key = 'schema_version'
+        """).fetchone()
+        stored = row[0] if row else None
+        artefacts = conn.execute("""
+            SELECT count(*) FROM sdmx.artefact
+        """).fetchone()
+    except duckdb.CatalogException:
+        return
+    n_artefacts = artefacts[0] if artefacts else 0
+    if stored == _SDMX_SCHEMA_VERSION or n_artefacts == 0:
+        return
+    msg = (
+        f"this catalog's SDMX structure is stored in sdmxlib schema {stored}, and this "
+        f"build of sdmxlib uses schema {_SDMX_SCHEMA_VERSION}. sdmxlib migrates by dropping "
+        f"the sdmx schema, which would delete the {n_artefacts} artefacts this catalog's "
+        f"datasets link to. Rebuild the catalog from its sources with this version, or "
+        f"open it with the statcatlib release that built it."
+    )
+    raise IncompatibleCatalogError(msg)
+
+
 @define(frozen=True)
 class EmbeddingMetadata:
     """Read-side view of a bundle's ``search/embeddings.json`` sidecar.
@@ -237,7 +292,7 @@ def _read_embeddings_meta(search_dir: Path) -> dict[str, object] | None:
         return None
     try:
         data = json.loads(meta_path.read_text(encoding="utf-8"))
-    except (OSError, json.JSONDecodeError):
+    except OSError, json.JSONDecodeError:
         return None
     if not isinstance(data, dict):
         return None
@@ -2677,6 +2732,16 @@ class Catalog:
 
             with Catalog("my_catalog").open() as cat:
                 ...
+
+        Raises `IncompatibleCatalogError` when the catalog's SDMX structure
+        was stored by an sdmxlib with another store schema, which this one
+        would drop on open. The connection is closed again first.
+
+        Returns:
+            This catalog, open.
+
+        Raises:
+            ValueError: ``embedding_model`` was given for an in-memory catalog.
         """
         if self._conn is not None:
             return self
@@ -2697,7 +2762,7 @@ class Catalog:
         # schema and the sdmx schema co-exist in one DuckDB file, so cross-
         # schema queries (e.g. join statcat.dataset to sdmx.codelist) work
         # natively without ATTACH plumbing.
-        self._registry = sdmxlib.LocalRegistry.from_connection(self._conn)
+        self._registry = self._open_registry(self._conn)
         # ATTACH the DuckLake sidecar (folder mode only — in-memory has no
         # lake). Sidecar layout because DuckDB rejects self-ATTACH; the
         # ducklake catalog file sits next to meta.duckdb so the bundle is
@@ -2737,6 +2802,28 @@ class Catalog:
                         self._query_prefix = meta.query_prefix
         return self
 
+    def _open_registry(self, conn: duckdb.DuckDBPyConnection) -> sdmxlib.LocalRegistry:
+        """Compose the sdmxlib `LocalRegistry` on *conn*, once the guard allows it.
+
+        Args:
+            conn: The connection `open` just made, which this catalog owns.
+
+        Returns:
+            The registry sharing *conn*.
+
+        Raises:
+            IncompatibleCatalogError: sdmxlib would drop this catalog's SDMX
+                structure. *conn* is closed first, releasing the file so the
+                release that built it can reopen it from this process.
+        """
+        try:
+            _refuse_sdmx_schema_rebuild(conn)
+        except IncompatibleCatalogError:
+            conn.close()
+            self._conn = None
+            raise
+        return sdmxlib.LocalRegistry.from_connection(conn)
+
     def _open_connection(self) -> tuple[duckdb.DuckDBPyConnection, Path | None]:
         """Open the DuckDB connection and (for folder mode) prepare the bundle dir.
 
@@ -2825,6 +2912,24 @@ class Catalog:
 
         ``embedding_model`` is rejected here because shared-connection mode
         has no on-disk ``data/`` directory to host the Lance index.
+
+        As `open` does, this raises `IncompatibleCatalogError` when the
+        connection's SDMX structure was stored by an sdmxlib with another
+        store schema, which this one would drop. *conn* is left open then:
+        the caller owns it.
+
+        Args:
+            conn: The connection to wrap. Left open by `close`.
+            embedding_model: Must be ``None``; see above.
+            document_prefix: Prefix for documents embedded into the index.
+            query_prefix: Prefix for queries embedded against the index.
+            read_only: Open the catalog schema without creating it.
+
+        Returns:
+            A catalog over *conn*, already open.
+
+        Raises:
+            ValueError: ``embedding_model`` was given.
         """
         if embedding_model is not None:
             msg = "from_connection() does not support embedding_model — Lance needs a folder-mode catalog"
@@ -2848,6 +2953,7 @@ class Catalog:
             _ = _catalog_schema.open_schema_readonly(conn)
         else:
             _ = _catalog_schema.create_schema(conn)
+        _refuse_sdmx_schema_rebuild(conn)
         instance._registry = sdmxlib.LocalRegistry.from_connection(conn)  # noqa: SLF001
         # Shared-connection mode: the caller owns any ATTACH plumbing. The
         # DataStore wraps the same connection without applying its own