statcatlib 0.9.1__tar.gz → 0.10.0__tar.gz

{statcatlib-0.9.1 → statcatlib-0.10.0}/LICENSE

@@ -1,3 +1,4 @@
+
                                  Apache License
                            Version 2.0, January 2004
                         http://www.apache.org/licenses/
@@ -32,9 +33,10 @@
       not limited to compiled object code, generated documentation,
       and conversions to other media types.
 
-      "Work" shall mean the work of authorship made available under
-      the License, as indicated by a copyright notice that is included in
-      or attached to the work (an example is provided in the Appendix below).
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
 
       "Derivative Works" shall mean any work, whether in Source or Object
       form, that is based on (or derived from) the Work and for which the
@@ -44,21 +46,23 @@
       separable from, or merely link (or bind by name) to the interfaces of,
       the Work and Derivative Works thereof.
 
-      "Contribution" shall mean, as submitted to the Licensor for inclusion
-      in the Work by the copyright owner or by an individual or Legal Entity
-      authorized to submit on behalf of the copyright owner. For the purposes
-      of this definition, "submitted" means any form of electronic, verbal,
-      or written communication sent to the Licensor or its representatives,
-      including but not limited to communication on electronic mailing lists,
-      source code control systems, and issue tracking systems that are managed
-      by, or on behalf of, the Licensor for the purpose of discussing and
-      improving the Work, but excluding communication that is conspicuously
-      marked or designated in writing by the copyright owner as "Not a
-      Contribution."
-
-      "Contributor" shall mean Licensor and any Legal Entity on behalf of
-      whom a Contribution has been received by the Licensor and included
-      within the Work.
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
 
    2. Grant of Copyright License. Subject to the terms and conditions of
       this License, each Contributor hereby grants to You a perpetual,
@@ -74,22 +78,22 @@
       use, offer to sell, sell, import, and otherwise transfer the Work,
       where such license applies only to those patent claims licensable
       by such Contributor that are necessarily infringed by their
-      Contribution(s) alone or by the combination of their Contribution(s)
+      Contribution(s) alone or by combination of their Contribution(s)
       with the Work to which such Contribution(s) was submitted. If You
-      institute patent litigation against any entity (including a cross-claim
-      or counterclaim in a lawsuit) alleging that the Work or any other
-      Contribution embodied in the Work constitutes direct or contributory
-      patent infringement, then any patent licenses granted to You under
-      this License for that Work shall terminate as of the date such
-      litigation is filed.
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
 
    4. Redistribution. You may reproduce and distribute copies of the
       Work or Derivative Works thereof in any medium, with or without
       modifications, and in Source or Object form, provided that You
       meet the following conditions:
 
-      (a) You must give any other recipients of the Work or Derivative
-          Works a copy of this License; and
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
 
       (b) You must cause any modified files to carry prominent notices
           stating that You changed the files; and
@@ -101,26 +105,28 @@
           the Derivative Works; and
 
       (d) If the Work includes a "NOTICE" text file as part of its
-          distribution, You must include a readable copy of the
-          attribution notices contained within such NOTICE file, in
-          at least one of the following places: within a NOTICE text
-          file distributed as part of the Derivative Works; within
-          the Source form or documentation, if provided along with the
-          Derivative Works; or, within a display generated by the
-          Derivative Works, if and wherever such third-party notices
-          normally appear. The contents of the NOTICE file are for
-          informational purposes only and do not modify the License.
-          You may add Your own attribution notices within Derivative
-          Works that You distribute, alongside or in addition to the
-          NOTICE text from the Work, provided that such additional
-          attribution notices cannot be construed as modifying the
-          License.
-
-      You may add Your own license statement for Your modifications and
-      may provide additional grant of rights to use, copy, modify, merge,
-      publish, distribute, sublicense, and/or sell copies of the
-      Contribution, either on its own behalf or on behalf of the Licensor,
-      and only if You accept these terms.
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
 
    5. Submission of Contributions. Unless You explicitly state otherwise,
       any Contribution intentionally submitted for inclusion in the Work
@@ -139,9 +145,9 @@
       agreed to in writing, Licensor provides the Work (and each
       Contributor provides its Contributions) on an "AS IS" BASIS,
       WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-      implied, including, without limitation, any conditions of TITLE,
-      NONINFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR
-      PURPOSE. You are solely responsible for determining the
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
       appropriateness of using or redistributing the Work and assume any
       risks associated with Your exercise of permissions under this License.
 
@@ -150,21 +156,37 @@
       unless required by applicable law (such as deliberate and grossly
       negligent acts) or agreed to in writing, shall any Contributor be
       liable to You for damages, including any direct, indirect, special,
-      incidental, or exemplary damages of any character arising as a
+      incidental, or consequential damages of any character arising as a
       result of this License or out of the use or inability to use the
-      Work (even if such Contributor has been advised of the possibility
-      of such damages).
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
 
    9. Accepting Warranty or Additional Liability. While redistributing
       the Work or Derivative Works thereof, You may choose to offer,
       and charge a fee for, acceptance of support, warranty, indemnity,
       or other liability obligations and/or rights consistent with this
-      License. However, in accepting such obligations, You may offer only
-      conditions consistent with this License, and charge a fee for,
-      such acceptance.
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
 
    END OF TERMS AND CONDITIONS
 
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
    Copyright 2026 gabrielgellner
 
    Licensed under the Apache License, Version 2.0 (the "License");

{statcatlib-0.9.1 → statcatlib-0.10.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: statcatlib
-Version: 0.9.1
+Version: 0.10.0
 Summary: Queryable open data catalog engine for DCAT-AP, StatDCAT-AP, CKAN, and SDMX
 Keywords: sdmx,dcat,open-data,catalog,duckdb,statistical-data,ckan
 Author: gabrielgellner
@@ -18,7 +18,7 @@ Classifier: Topic :: Internet :: WWW/HTTP :: Indexing/Search
 Requires-Dist: attrs>=26.1.0
 Requires-Dist: duckdb>=2.0.0
 Requires-Dist: httpx>=0.28
-Requires-Dist: sdmxlib>=0.70.1
+Requires-Dist: sdmxlib>=0.72.0
 Requires-Dist: lancedb>=0.39 ; extra == 'search'
 Requires-Dist: sentence-transformers>=3.0 ; extra == 'search'
 Requires-Dist: typer>=0.15 ; extra == 'search'
@@ -39,7 +39,9 @@ Description-Content-Type: text/markdown
 > take that into account before adopting it.
 
 **statcatlib** is a Python library for building, managing, and querying
-statistical metadata catalogs. Formerly published as `statcatlib`.
+statistical metadata catalogs. Formerly published as `pinaxlib`.
+
+**Documentation:** <https://statcat-suite.github.io/statcatlib/>
 
 Inspired by DCAT and the broader landscape of metadata standards (SDMX, DDI),
 statcatlib provides a general-purpose engine for catalog storage, discovery, and

{statcatlib-0.9.1 → statcatlib-0.10.0}/README.md

@@ -5,7 +5,9 @@
 > take that into account before adopting it.
 
 **statcatlib** is a Python library for building, managing, and querying
-statistical metadata catalogs. Formerly published as `statcatlib`.
+statistical metadata catalogs. Formerly published as `pinaxlib`.
+
+**Documentation:** <https://statcat-suite.github.io/statcatlib/>
 
 Inspired by DCAT and the broader landscape of metadata standards (SDMX, DDI),
 statcatlib provides a general-purpose engine for catalog storage, discovery, and

{statcatlib-0.9.1 → statcatlib-0.10.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "statcatlib"
-version = "0.9.1"
+version = "0.10.0"
 description = "Queryable open data catalog engine for DCAT-AP, StatDCAT-AP, CKAN, and SDMX"
 license = "Apache-2.0"
 license-files = ["LICENSE"]
@@ -29,7 +29,7 @@ dependencies = [
     "attrs>=26.1.0",
     "duckdb>=2.0.0",
     "httpx>=0.28",
-    "sdmxlib>=0.70.1",
+    "sdmxlib>=0.72.0",
 ]
 
 [[project.authors]]

{statcatlib-0.9.1 → statcatlib-0.10.0}/pyproject.toml.orig

@@ -1,6 +1,6 @@
 [project]
 name = "statcatlib"
-version = "0.9.1"
+version = "0.10.0"
 description = "Queryable open data catalog engine for DCAT-AP, StatDCAT-AP, CKAN, and SDMX"
 license = "Apache-2.0"
 license-files = ["LICENSE"]
@@ -30,8 +30,9 @@ dependencies = [
     # >=0.70: needs Python 3.14 and DuckDB 2.0; `reg.data()` is gone, and a
     # component without a concept is refused as it is built. >=0.70.1:
     # `reg.obs()` asks Eurostat for CSV it serves, so `ingest_data()` can
-    # fetch from it (sdmxlib GH-50).
-    "sdmxlib>=0.70.1",
+    # fetch from it (sdmxlib GH-50). >=0.72: `sdmxlib.bundle`, the manifest
+    # and checker `publish_bundle` and `check_bundle` build on (GH-3).
+    "sdmxlib>=0.72.0",
 ]
 
 [project.optional-dependencies]

{statcatlib-0.9.1 → statcatlib-0.10.0}/src/statcatlib/__init__.py

@@ -81,6 +81,7 @@ from statcatlib.store import (
     DimensionsScope,
     EmbeddingMetadata,
     Hit,
+    SEARCH_ANNOTATION_TYPES,
     SEARCH_CONTEXT_ANNOTATION_TYPE,
     SEARCH_EXCLUDE_ANNOTATION_TYPE,
     IncompatibleCatalogError,
@@ -98,6 +99,7 @@ from statcatlib.store import (
 from statcatlib.store.sentinel import NotLoadedError, is_unloaded
 
 __all__ = [
+    "SEARCH_ANNOTATION_TYPES",
     "SEARCH_CONTEXT_ANNOTATION_TYPE",
     "SEARCH_EXCLUDE_ANNOTATION_TYPE",
     "VALID_INCLUDE_RELS",

{statcatlib-0.9.1 → statcatlib-0.10.0}/src/statcatlib/bundle.py

@@ -33,25 +33,50 @@ assumed (#120):
   stamp resolves rather than leaving it to DuckDB to discover.
 
 See #94, #98 and #120 for the design history.
+
+A bundle is published by writing sdmxlib's ``bundle.manifest.json`` last.
+`publish_bundle` writes it with a ``statcat`` section describing the catalog
+and its search index, and `check_bundle` runs sdmxlib's checks and then
+`check_catalog` against that section (GH-3).
 """
 
 from __future__ import annotations
 
 import errno
+import json
 import shutil
+from collections.abc import Collection, Mapping
 from contextlib import contextmanager
+from importlib import metadata
 from pathlib import Path
 from typing import TYPE_CHECKING, Literal
 
 import duckdb
 from attrs import field, frozen
+from sdmxlib.bundle import (
+    MANIFEST_NAME,
+    Finding,
+    Manifest,
+    Overlay,
+    OverlayEntry,
+    Producer,
+    Provenance,
+    Report,
+    write_manifest,
+)
+from sdmxlib.bundle import check_bundle as sdmx_check_bundle
 
 if TYPE_CHECKING:
-    from collections.abc import Generator
+    from collections.abc import Generator, Iterator
+
+    from sdmxlib import LocalRegistry
 
     from statcatlib.store import Catalog
+    from statcatlib.store.lance import LanceIndex, LanguageLayout
 
 __all__ = [
+    "STATCAT_SECTION",
+    "STATCAT_SECTION_FORMAT",
     "At",
     "Auto",
     "BundleDataPathError",
@@ -59,9 +84,14 @@ __all__ = [
     "Colocated",
     "DataPolicy",
     "Embedded",
+    "catalog_section",
+    "check_bundle",
+    "check_catalog",
     "inspect_bundle",
     "normalize_lake_data_path",
     "open_bundle",
+    "overlay_affects_search",
+    "publish_bundle",
     "read_current_data_path",
 ]
 
@@ -563,3 +593,294 @@ def open_bundle(
     )
     with catalog.open() as opened:
         yield opened
+
+
+# -- the published bundle: statcatlib's part of sdmxlib's contract -------
+#
+# sdmxlib owns ``bundle.manifest.json`` (sdmxlib GH-56): the token caches key
+# on, a sha256 per file, and the checker. A statcat bundle is an sdmxlib
+# bundle plus the ``statcat`` schema and ``search/``, so statcatlib adds what
+# only it can describe: a ``statcat`` section in the manifest, and checks
+# that the catalog and its search index are what that section says (GH-3).
+
+#: The manifest's extension section statcatlib owns.
+STATCAT_SECTION = "statcat"
+
+#: Version of the ``statcat`` section. Bumped when its fields change meaning.
+#: 2 added ``search.annotations``, the annotations the index was built from.
+STATCAT_SECTION_FORMAT = 2
+
+_SEARCH_DIRNAME = "search"
+_INDEX_MANIFEST = "index.json"
+
+
+def _has_index(search_dir: Path) -> bool:
+    """Whether ``search/`` holds anything. Opening a folder catalog creates it empty."""
+    return search_dir.is_dir() and any(search_dir.iterdir())
+
+
+def _search_section(cat: Catalog, search_dir: Path) -> dict[str, object] | None:
+    """The ``search`` part of the section.
+
+    Args:
+        cat: The bundle's catalog, open.
+        search_dir: Its ``search/`` directory.
+
+    Returns:
+        The index format version, embedding model and per-language layout,
+        or ``None`` when no index is built.
+
+    Raises:
+        ValueError: ``search/`` holds an index from before per-language
+            tables, which has no layout to record.
+    """
+    if not _has_index(search_dir):
+        return None
+    layout = cat.index_layout
+    if not layout:
+        msg = f"{search_dir} holds a search index with no {_INDEX_MANIFEST}: rebuild it with build_index() first"
+        raise ValueError(msg)
+    index_manifest = json.loads((search_dir / _INDEX_MANIFEST).read_text())
+    return {
+        "index_version": index_manifest.get("version"),
+        "embedding_model": cat.embedding_model_name,
+        "languages": {lang: {"rows": entry["rows"], "stemmer": entry["stemmer"]} for lang, entry in layout.items()},
+        "annotations": cat.index_annotations,
+    }
+
+
+def catalog_section(path: str | Path) -> dict[str, object]:
+    """Describe the catalog in a bundle, as its manifest's ``statcat`` section.
+
+    Opens the bundle read-only. The section records the number of catalog
+    datasets and, when a search index is built, its index format version,
+    embedding model, and each language's row count and stemmer.
+
+    Args:
+        path: The bundle directory.
+
+    Returns:
+        The section, ready for ``write_manifest(extensions=...)``.
+    """
+    from statcatlib.store import Catalog  # noqa: PLC0415
+
+    root = Path(path)
+    with Catalog(root, read_only=True, data_path=Embedded()) as cat:
+        row = cat.connection.execute("""SELECT count(*) FROM statcat.dataset""").fetchone()
+        return {
+            "format": STATCAT_SECTION_FORMAT,
+            "statcatlib_version": metadata.version("statcatlib"),
+            "datasets": row[0] if row else 0,
+            "search": _search_section(cat, root / _SEARCH_DIRNAME),
+        }
+
+
+def publish_bundle(
+    path: str | Path,
+    *,
+    bundle_id: str,
+    producer: Producer,
+    product: dict[str, object] | None = None,
+    extensions: dict[str, dict[str, object]] | None = None,
+    provenance: Provenance | None = None,
+) -> Manifest:
+    """Publish a statcat bundle: sdmxlib's manifest, with the ``statcat`` section.
+
+    `sdmxlib.bundle.write_manifest` with `catalog_section` added to its
+    extensions. Call it **last**, after `Catalog.build_index`, once nothing
+    in the directory will change: the manifest lists every file with its
+    sha256, so a file written afterwards fails `check_bundle`. Publishing
+    again mints a new token, which is what tells caches to refresh.
+
+    Args:
+        path: The bundle directory.
+        bundle_id: The producer's stable name for the bundle, kept across rebuilds.
+        producer: What built it.
+        product: Producer-specific fields, carried and not interpreted.
+        extensions: Other packages' sections; ``statcat`` is filled in here.
+        provenance: Where a fetched bundle came from.
+
+    Returns:
+        The manifest written.
+
+    Raises:
+        ValueError: *extensions* already has a ``statcat`` section, or the
+            search index predates per-language tables and must be rebuilt.
+    """
+    if extensions and STATCAT_SECTION in extensions:
+        msg = f"the {STATCAT_SECTION!r} section is written by publish_bundle, not passed to it"
+        raise ValueError(msg)
+    section = catalog_section(path)
+    return write_manifest(
+        path,
+        bundle_id=bundle_id,
+        producer=producer,
+        product=product,
+        extensions={**(extensions or {}), STATCAT_SECTION: section},
+        provenance=provenance,
+    )
+
+
+def _check_catalog_rows(conn: duckdb.DuckDBPyConnection, section: Mapping[str, object]) -> Iterator[Finding]:
+    """The catalog's datasets: counted as declared, each linked flow stored."""
+    row = conn.execute("""SELECT count(*) FROM statcat.dataset""").fetchone()
+    found = row[0] if row else 0
+    if found != section.get("datasets"):
+        detail = f"{found} datasets, the manifest says {section.get('datasets')}"
+        yield Finding(STATCAT_SECTION, "statcat.dataset", detail)
+    dangling = conn.execute(
+        """
+        SELECT d.identifier, d.sdmx_dataflow_urn
+        FROM statcat.dataset d
+        WHERE d.sdmx_dataflow_urn IS NOT NULL
+          AND NOT EXISTS (SELECT 1 FROM sdmx.dataflow f WHERE f.urn = d.sdmx_dataflow_urn)
+        ORDER BY d.identifier
+        """
+    ).fetchall()
+    for identifier, urn in dangling:
+        yield Finding(STATCAT_SECTION, identifier, f"links {urn}, which the registry does not hold")
+
+
+def _check_index_annotations(conn: duckdb.DuckDBPyConnection, declared: object) -> Iterator[Finding]:
+    """The store holds the annotations the index was built from: no overlay folded in since."""
+    from statcatlib.store.store import annotation_fingerprint  # noqa: PLC0415
+
+    subject = f"{_SEARCH_DIRNAME} annotations"
+    if not isinstance(declared, Mapping) or not isinstance(types := declared.get("types"), list):
+        yield Finding(
+            STATCAT_SECTION, subject, "the index records no annotation fingerprint: rebuild with build_index()"
+        )
+        return
+    if annotation_fingerprint(conn, [str(t) for t in types]) != declared.get("fingerprint"):
+        detail = (
+            f"annotations of type {', '.join(map(str, types))} differ from those the index was built from "
+            "(an overlay folded in?): rebuild with build_index()"
+        )
+        yield Finding(STATCAT_SECTION, subject, detail)
+
+
+def _check_search(root: Path, declared: object) -> Iterator[Finding]:
+    """The search index: present exactly when declared, with the declared layout."""
+    search_dir = root / _SEARCH_DIRNAME
+    if declared is None:
+        if _has_index(search_dir):
+            yield Finding(STATCAT_SECTION, _SEARCH_DIRNAME, "a search index the manifest does not declare")
+        return
+    if not isinstance(declared, Mapping) or not isinstance(languages := declared.get("languages"), Mapping):
+        yield Finding(STATCAT_SECTION, MANIFEST_NAME, "the search section has no languages")
+        return
+    if not (search_dir / _INDEX_MANIFEST).is_file():
+        yield Finding(STATCAT_SECTION, f"{_SEARCH_DIRNAME}/{_INDEX_MANIFEST}", "the manifest declares a search index")
+        return
+    try:
+        from statcatlib.store.lance import LanceIndex  # noqa: PLC0415
+    except ImportError:
+        yield Finding(STATCAT_SECTION, _SEARCH_DIRNAME, "cannot check the search index without statcatlib[search]")
+        return
+    index = LanceIndex(search_dir)
+    layout = index.describe()
+    for lang in sorted(set(languages) | set(layout)):
+        yield from _check_language(index, lang, layout.get(lang), languages.get(lang))
+
+
+def _check_language(index: LanceIndex, lang: str, found: LanguageLayout | None, want: object) -> Iterator[Finding]:
+    """One language's table: declared, stemmed as declared, holding the declared rows."""
+    subject = f"{_SEARCH_DIRNAME} [{lang}]"
+    if found is None:
+        yield Finding(STATCAT_SECTION, subject, "declared in the manifest, absent from the index")
+        return
+    if not isinstance(want, Mapping):
+        yield Finding(STATCAT_SECTION, subject, "in the index, not declared in the manifest")
+        return
+    if found["stemmer"] != want.get("stemmer"):
+        yield Finding(
+            STATCAT_SECTION, subject, f"stemmed with {found['stemmer']}, the manifest says {want.get('stemmer')}"
+        )
+    table = index.table_for(lang)
+    rows = table.count_rows() if table is not None else 0
+    if rows != want.get("rows"):
+        yield Finding(STATCAT_SECTION, subject, f"{rows} rows, the manifest says {want.get('rows')}")
+
+
+def check_catalog(root: Path, manifest: Manifest, reg: LocalRegistry) -> list[Finding]:
+    """The ``statcat`` check `check_bundle` adds to sdmxlib's.
+
+    - the ``statcat`` schema is in ``meta.duckdb``, and the manifest has a
+      ``statcat`` section in a format this build reads;
+    - the catalog holds the datasets the section counts, and every dataset
+      linked to a dataflow names one the registry holds;
+    - ``search/`` is present exactly when the section declares an index,
+      with the declared languages, each stemmed as declared and holding the
+      declared rows;
+    - the store's annotations of the types the index read are the ones it was
+      built from, so an overlay folded in since shows up as a stale index.
+
+    Args:
+        root: The bundle directory.
+        manifest: Its manifest.
+        reg: sdmxlib's registry over the bundle.
+
+    Returns:
+        Everything found wrong with the catalog's part of the bundle.
+    """
+    conn = reg.connection
+    has_schema = conn.execute(
+        """SELECT count(*) FROM information_schema.tables WHERE table_schema = 'statcat' AND table_name = 'dataset'"""
+    ).fetchone()
+    if not has_schema or not has_schema[0]:
+        return [Finding(STATCAT_SECTION, "meta.duckdb", "no statcat schema: not a statcat catalog")]
+    section = manifest.extensions.get(STATCAT_SECTION)
+    if section is None:
+        return [Finding(STATCAT_SECTION, MANIFEST_NAME, "no statcat section: publish with statcatlib's publish_bundle")]
+    if section.get("format") != STATCAT_SECTION_FORMAT:
+        detail = f"statcat section format {section.get('format')!r}, this build reads {STATCAT_SECTION_FORMAT}"
+        return [Finding(STATCAT_SECTION, MANIFEST_NAME, detail)]
+    findings = [*_check_catalog_rows(conn, section), *_check_search(root, section.get("search"))]
+    search = section.get("search")
+    if isinstance(search, Mapping):
+        findings += _check_index_annotations(conn, search.get("annotations"))
+    return findings
+
+
+def check_bundle(path: str | Path, *, references: bool = True) -> Report:
+    """Check a statcat bundle: sdmxlib's checks, then the catalog's.
+
+    sdmxlib's `sdmxlib.bundle.check_bundle` checks the manifest, the files
+    against their checksums, the lake, the bindings and the structural
+    closure; `check_catalog` runs after them as the ``statcat`` check. Run
+    it where a bundle is built or received, never on the path that serves
+    it. ``statcat bundle check`` runs it from a shell.
+
+    Args:
+        path: The bundle directory.
+        references: Whether to check the structural closure, the costly check.
+
+    Returns:
+        What was checked and what was found.
+    """
+    return sdmx_check_bundle(path, references=references, extensions={STATCAT_SECTION: check_catalog})
+
+
+def overlay_affects_search(overlay: Overlay, types: Collection[str] | None = None) -> tuple[OverlayEntry, ...]:
+    """The entries of *overlay* that change what the search index should hold.
+
+    An overlay applied at serve time never reaches statcatlib's index, and one
+    folded into a bundle needs ``build_index()`` before search reflects it.
+    Either way, an entry acting on an annotation type the index reads is a
+    change search will not see on its own. Call this where overlays are
+    loaded, to log or refuse them. Static: it needs no bundle.
+
+    Args:
+        overlay: The overlay document.
+        types: The annotation types the index reads. ``None`` means
+            `SEARCH_ANNOTATION_TYPES`, what `Catalog.build_index` reads by
+            default; pass ``cat.index_annotations["types"]`` for an index
+            built otherwise.
+
+    Returns:
+        Those entries, in document order; empty, and so falsy, when there are none.
+    """
+    from statcatlib.store.store import SEARCH_ANNOTATION_TYPES  # noqa: PLC0415
+
+    wanted = set(SEARCH_ANNOTATION_TYPES if types is None else types)
+    return tuple(entry for entry in overlay.entries if entry.type in wanted)

{statcatlib-0.9.1 → statcatlib-0.10.0}/src/statcatlib/cli/__init__.py

@@ -225,6 +225,39 @@ def bundle_explain(
         raise typer.Exit(code=1)
 
 
+@bundle_app.command("check")
+def bundle_check(
+    bundle: Path = typer.Argument(
+        ...,
+        help="Path to a published statcatlib Catalog bundle (the folder containing bundle.manifest.json).",
+        exists=True,
+        file_okay=False,
+        dir_okay=True,
+    ),
+    references: bool = typer.Option(
+        True,
+        "--references/--no-references",
+        help="Check that every structural reference resolves in the bundle. The costly check.",
+    ),
+) -> None:
+    """Check a published bundle against its manifest: sdmxlib's checks, then the catalog's.
+
+    Run it where a bundle is built or received, not on the serving path.
+    Lists everything found wrong and exits 1 if anything was.
+    """
+    from statcatlib.bundle import check_bundle  # noqa: PLC0415
+
+    report = check_bundle(bundle.resolve(), references=references)
+    for finding in report.findings:
+        # The finding's own "[check]" prefix is text, not Rich markup.
+        console.print(str(finding), style="red", markup=False, highlight=False)
+    checked = ", ".join(report.checked)
+    if not report.ok:
+        console.print(f"[red]{len(report.findings)} finding(s)[/] — checked {checked}")
+        raise typer.Exit(code=1)
+    console.print(f"[green]ok[/] — checked {checked}")
+
+
 @bundle_app.command("attach-cmd")
 def bundle_attach_cmd(
     bundle: Path = typer.Argument(

{statcatlib-0.9.1 → statcatlib-0.10.0}/src/statcatlib/sources/sdmx.py

@@ -17,6 +17,7 @@ from statcatlib.model.dataset import AggregateDataset
 
 if TYPE_CHECKING:
     from sdmxlib.api.registry import RestRegistry
+    from sdmxlib.local.registry import LocalRegistry
     from sdmxlib.model.in_memory_registry import InMemoryRegistry
 
     from statcatlib.model.skos import ConceptRef
@@ -133,7 +134,7 @@ def dataset_from_dataflow(
 
 
 def prepare_sdmx(
-    registry: InMemoryRegistry,
+    registry: InMemoryRegistry | LocalRegistry,
     *,
     publisher: Agent | None = None,
     themes: list[ConceptRef] | None = None,
@@ -148,6 +149,9 @@ def prepare_sdmx(
 
     Code and codelist information is resolved at query time via the SDMX
     structural tables once ``local.ingest(registry)`` has been called.
+
+    *registry* can also be a `LocalRegistry` whose store already holds the
+    structures, such as a bundle's: each flow's DSD is read from the store.
     """
     results: list[AggregateDataset] = []
     for flow in registry.get_all(Dataflow):

{statcatlib-0.9.1 → statcatlib-0.10.0}/src/statcatlib/store/__init__.py

@@ -26,6 +26,7 @@ from statcatlib.store.store import (
     DatasetsScope,
     EmbeddingMetadata,
     Hit,
+    SEARCH_ANNOTATION_TYPES,
     SEARCH_CONTEXT_ANNOTATION_TYPE,
     SEARCH_EXCLUDE_ANNOTATION_TYPE,
     IncompatibleCatalogError,

{statcatlib-0.9.1 → statcatlib-0.10.0}/src/statcatlib/store/lance.py

@@ -147,6 +147,17 @@ def _as_opt_str(value: object) -> str | None:
     return value if isinstance(value, str) else None
 
 
+def _as_annotations(entry: object) -> IndexAnnotations | None:
+    """Narrow the manifest's ``annotations`` entry, or ``None`` if absent or malformed."""
+    if not isinstance(entry, Mapping):
+        return None
+    types = entry.get("types")
+    fingerprint = _as_opt_str(entry.get("fingerprint"))
+    if not isinstance(types, list) or fingerprint is None:
+        return None
+    return {"types": [str(t) for t in types], "fingerprint": fingerprint}
+
+
 def _as_layout(entry: object) -> LanguageLayout | None:
     """Narrow one manifest ``languages`` entry, or ``None`` if it is not one.
 
@@ -263,6 +274,20 @@ class LanguageLayout(TypedDict):
     stemmer: str | None
 
 
+class IndexAnnotations(TypedDict):
+    """The annotations an index was built from, as `Catalog.index_annotations` reports it.
+
+    `types` are the annotation types `build_index` read (what to exclude, what
+    context and synonyms to fold in); `fingerprint` is a digest of every
+    annotation of those types in the store at the time. An overlay folded
+    into the store afterwards changes the fingerprint the store yields, which
+    is how a stale index is detected without rebuilding it (GH-3).
+    """
+
+    types: list[str]
+    fingerprint: str
+
+
 class _Manifest(TypedDict):
     """``index.json``, as written.
 
@@ -277,6 +302,7 @@ class _Manifest(TypedDict):
     written_by: str
     generation: int
     languages: dict[str, LanguageLayout]
+    annotations: NotRequired[IndexAnnotations]
 
 
 class SearchRow(TypedDict):
@@ -723,6 +749,27 @@ class _LanguageIndex:
         self._ann_index_type: str | None = None
         if name in db.list_tables().tables:
             self._tbl = db.open_table(name)
+            self._has_fts = self._fts_covers_table()
+
+    def _fts_covers_table(self) -> bool:
+        """Whether the opened table already has an FTS index over every row.
+
+        Without this, the first BM25 query in each process rebuilt the
+        index with ``replace=True``: every serving worker re-indexed on
+        startup, and a read-only open wrote new index files into a
+        published bundle, breaking the checksums in its manifest.
+        Rows added after the index was built leave it partial, and then
+        a rebuild is still due.
+        """
+        assert self._tbl is not None  # noqa: S101
+        try:
+            indices = self._tbl.list_indices()
+        except Exception:  # noqa: BLE001
+            return False
+        return any(
+            i.index_type == "FTS" and list(i.columns) == ["text"] and not getattr(i, "num_unindexed_rows", 0)
+            for i in indices
+        )
 
     @property
     def table(self) -> lancedb.table.Table | None:
@@ -964,11 +1011,13 @@ class LanceIndex:
         self._langs: dict[str, _LanguageIndex] = {}
         self._legacy: lancedb.table.Table | None = None
         self._generation = 0
+        self._annotations: IndexAnnotations | None = None
 
         names = set(self._db.list_tables().tables)
         manifest = self._read_manifest()
         if manifest is not None:
             self._generation = _as_int(manifest.get("generation"))
+            self._annotations = _as_annotations(manifest.get("annotations"))
             languages = manifest.get("languages")
             entries = languages.items() if isinstance(languages, Mapping) else ()
             for lang, entry in entries:
@@ -1088,6 +1137,8 @@ class LanceIndex:
                 for lang, index in sorted(indexes.items())
             },
         }
+        if self._annotations is not None:
+            payload["annotations"] = self._annotations
         self._path.mkdir(parents=True, exist_ok=True)
         tmp = self._path / f"{_MANIFEST_FILE}.tmp"
         tmp.write_text(json.dumps(payload, indent=2, sort_keys=True))
@@ -1105,6 +1156,11 @@ class LanceIndex:
             return _LanguageIndex(self._db, slug, _TABLE, filter_lang=True)
         return None
 
+    @property
+    def annotations(self) -> IndexAnnotations | None:
+        """The annotations this index was built from; ``None`` for one built before they were recorded."""
+        return self._annotations
+
     @property
     def languages(self) -> list[str]:
         """Languages with their own table, sorted."""
@@ -1182,7 +1238,13 @@ class LanceIndex:
         if added_language:
             self._write_manifest(self._langs, self._generation)
 
-    def add_bulk(self, rows: list[SearchRow], *, embedding_dim: int | None = None) -> None:
+    def add_bulk(
+        self,
+        rows: list[SearchRow],
+        *,
+        embedding_dim: int | None = None,
+        annotations: IndexAnnotations | None = None,
+    ) -> None:
         """Bulk-load rows, replacing every language's table.
 
         More efficient than per-row ``add()`` for full rebuilds — avoids
@@ -1203,6 +1265,9 @@ class LanceIndex:
                 null-typed embedding column, and the ANN index would
                 silently never build. ``None`` falls back to scanning the
                 rows, which is correct whenever at least one is embedded.
+            annotations: The annotations *rows* were built from. Recorded in
+                the same manifest write that commits the new generation, so
+                the two cannot disagree after a crash.
 
         Raises:
             Exception: Whatever the underlying build raised. Every table
@@ -1233,6 +1298,7 @@ class LanceIndex:
                 index.drop()
             raise
 
+        self._annotations = annotations
         self._write_manifest(built, generation)
 
         # Past this point the rebuild has committed; everything below is

{statcatlib-0.9.1 → statcatlib-0.10.0}/src/statcatlib/store/store.py

@@ -18,7 +18,14 @@ from typing import TYPE_CHECKING, Any, NamedTuple, Protocol, Self, TypedDict, ov
 if TYPE_CHECKING:
     from collections.abc import Callable, Generator, Sequence
 
-    from statcatlib.store.lance import Bm25HitRow, LanceIndex, LanguageLayout, SearchRow, VectorHitRow
+    from statcatlib.store.lance import (
+        Bm25HitRow,
+        IndexAnnotations,
+        LanceIndex,
+        LanguageLayout,
+        SearchRow,
+        VectorHitRow,
+    )
 
 import duckdb
 import sdmxlib
@@ -1597,6 +1604,75 @@ SEARCH_CONTEXT_ANNOTATION_TYPE = "statcat:searchContext"
 #: structural tables — only its search rows are skipped.
 SEARCH_EXCLUDE_ANNOTATION_TYPE = "statcat:searchExclude"
 
+#: The SKOS synonym annotation `build_index` folds into code rows
+#: (``include_alt_labels``).
+_ALT_LABEL_ANNOTATION_TYPE = "skos:altLabel"
+
+#: Every annotation type `build_index` reads by default. An annotation
+#: overlay touching one of these changes what the index should hold, so it
+#: needs ``build_index()`` to take effect in search (GH-3).
+SEARCH_ANNOTATION_TYPES: tuple[str, ...] = (
+    SEARCH_CONTEXT_ANNOTATION_TYPE,
+    SEARCH_EXCLUDE_ANNOTATION_TYPE,
+    _ALT_LABEL_ANNOTATION_TYPE,
+)
+
+
+def annotation_fingerprint(conn: duckdb.DuckDBPyConnection, types: Sequence[str]) -> str:
+    """A digest of every annotation of *types* in the ``sdmx`` store.
+
+    What `build_index` records, and what a stale-index check recomputes: it
+    changes when an annotation of one of *types* is added, removed or edited
+    anywhere in the store — on any owner, which over-approximates the codes
+    the index reads and so never misses a change. Each annotation row is
+    hashed with its owner, position, id, title and value, and each of its
+    localised texts with its owner, position and language.
+
+    The row hashes are summed, not sorted and concatenated: order-independent
+    without a sort, so it costs one scan (about 140 ms for 400,000
+    annotations with 400,000 texts). md5 rather than DuckDB's ``hash``, which
+    is not stable across DuckDB versions. NULLs are spelled out, since
+    ``concat_ws`` skips them and would let a title and a value trade places.
+
+    Args:
+        conn: A connection whose ``sdmx`` schema holds the annotations.
+        types: The annotation types to cover.
+
+    Returns:
+        A hex digest. With *types* empty, or no ``sdmx`` schema, the digest
+        of nothing.
+    """
+    nothing = hashlib.md5(b"", usedforsecurity=False).hexdigest()
+    has_sdmx = conn.execute(
+        """SELECT count(*) FROM information_schema.tables WHERE table_schema = 'sdmx' AND table_name = 'annotation'"""
+    ).fetchone()
+    if not types or not has_sdmx or not has_sdmx[0]:
+        return nothing
+    row = conn.execute(
+        """
+        WITH typed AS (
+            SELECT id, owner_urn, ordinal, annotation_id, annotation_type, title, value
+            FROM sdmx.annotation
+            WHERE annotation_type IN (SELECT UNNEST($types))
+        )
+        SELECT
+            (SELECT count(*) FROM typed),
+            (SELECT sum(md5_number_lower(concat_ws(chr(31),
+                owner_urn, ordinal, coalesce(annotation_id, chr(0)), annotation_type,
+                coalesce(title, chr(0)), coalesce(value, chr(0))))) FROM typed),
+            (SELECT count(*) FROM typed a JOIN sdmx.annotation_text t ON t.annotation_id = a.id),
+            (SELECT sum(md5_number_lower(concat_ws(chr(31),
+                a.owner_urn, a.ordinal, coalesce(t.lang, chr(0)), coalesce(t.text, chr(0)))))
+             FROM typed a JOIN sdmx.annotation_text t ON t.annotation_id = a.id)
+        """,
+        {"types": sorted(types)},
+    ).fetchone()
+    if row is None:
+        return nothing
+    folded = "|".join(str(part) for part in (sorted(types), *row))
+    return hashlib.md5(folded.encode(), usedforsecurity=False).hexdigest()
+
+
 #: Bucket key for search text that carries no language. Not a language
 #: tag: it means "applies to every language", which is the only thing
 #: untagged text can mean. Text bucketed here folds into every language's
@@ -2785,6 +2861,7 @@ class Catalog:
             self._data = sdmxlib.DuckDBDataStore.from_registry(self._registry)
         if self._bundle_dir is not None:
             self._init_lance()
+            self._warn_if_index_annotations_changed()
             # Restore embedding metadata written by a previous build_index()
             # so the caller doesn't have to remember which model + prefixes
             # match this index. Missing/malformed metadata leaves the store
@@ -2982,6 +3059,35 @@ class Catalog:
         except ImportError:
             return
 
+    def _warn_if_index_annotations_changed(self) -> None:
+        """Warn if the store's search annotations are not the ones the index was built from.
+
+        A warning, never an error and never a rebuild: opening a bundle in a
+        serving pod must not fail or write because an overlay was folded in.
+        It fires on every open until ``build_index()`` is run (GH-3).
+        """
+        recorded = self._lance.annotations if self._lance is not None else None
+        if recorded is None or self._conn is None:
+            return
+        if annotation_fingerprint(self._conn, recorded["types"]) == recorded["fingerprint"]:
+            return
+        warnings.warn(
+            f"search index is out of date: annotations of type {', '.join(recorded['types'])} have changed "
+            "since it was built (an overlay folded into the store?), so search does not reflect them. "
+            "Rebuild with build_index() to fix. (statcatlib GH-3)",
+            stacklevel=3,
+        )
+
+    @property
+    def index_annotations(self) -> IndexAnnotations | None:
+        """The annotation types the search index was built from, and their fingerprint.
+
+        ``None`` when no index is built, or for one built before statcatlib
+        recorded them. Compare ``types`` with an annotation overlay's entries
+        to tell whether applying it needs a ``build_index()`` to reach search.
+        """
+        return self._lance.annotations if self._lance is not None else None
+
     @property
     def connection(self) -> duckdb.DuckDBPyConnection:
         """The underlying DuckDB connection (for sharing with sdmxlib LocalRegistry).
@@ -4625,9 +4731,18 @@ class Catalog:
         )
 
         embedding_dim = _infer_embedding_dim(self._embedding_model, dataset_rows)
+        # The annotation types this build read, and a digest of them, so a
+        # later change to them (a folded overlay) is detected, not missed.
+        annotation_types: set[str] = set()
+        if include_codes:
+            annotation_types |= {*context_annotation_types, *exclude_annotation_types}
+            if include_alt_labels:
+                annotation_types.add(_ALT_LABEL_ANNOTATION_TYPE)
+        types = sorted(annotation_types)
         self._lance.add_bulk(
             dataset_rows + code_rows + concept_rows + refmeta_rows,
             embedding_dim=embedding_dim,
+            annotations={"types": types, "fingerprint": annotation_fingerprint(self._c, types)},
         )
         self._lance.build_index()
         if progress is not None:

statcatlib-0.10.0/src/statcatlib/testing.py

@@ -0,0 +1,72 @@
+"""Synthetic statcat bundles, for tests and benchmarks at any shape (GH-3).
+
+`sdmxlib.testing.synthetic_bundle` builds a conforming sdmxlib bundle from a
+`sdmxlib.testing.Shape` and a seed. `synthetic_catalog` makes it a statcat
+bundle: a catalog dataset per dataflow, a search index over it, and a
+manifest with the ``statcat`` section, so it passes
+`statcatlib.bundle.check_bundle`. The same seed builds the same structures and
+observations, so a failure reproduces.
+
+Example:
+    from sdmxlib.testing import Dim, Shape
+
+    from statcatlib.testing import synthetic_catalog
+
+    shape = Shape(dimensions=(Dim(codes=5000, depth=4), Dim(codes=12)), observations=100_000, languages=("en", "fr"))
+    synthetic_catalog("out/stress", shape, seed=7)
+"""
+
+from __future__ import annotations
+
+from importlib import metadata
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+from sdmxlib.bundle import Producer
+from sdmxlib.testing import synthetic_bundle
+
+from statcatlib.bundle import Embedded, publish_bundle
+from statcatlib.sources.sdmx import prepare_sdmx
+from statcatlib.store import Catalog
+
+if TYPE_CHECKING:
+    from sdmxlib.bundle import Manifest
+    from sdmxlib.testing import Shape
+
+__all__ = ["synthetic_catalog"]
+
+
+def synthetic_catalog(
+    path: str | Path,
+    shape: Shape,
+    *,
+    seed: int = 0,
+    index: bool = True,
+    bundle_id: str = "synthetic",
+) -> Manifest:
+    """Build a published statcat bundle of the given shape at *path*.
+
+    The structures and observations are `sdmxlib.testing.synthetic_bundle`'s.
+    Each dataflow becomes an `AggregateDataset`, the search index is built
+    unless *index* is false (it needs ``statcatlib[search]``), and the bundle
+    is published last with `statcatlib.bundle.publish_bundle`.
+
+    Args:
+        path: The bundle directory to create. It must not hold a bundle already.
+        shape: Codelist sizes, hierarchy depth, observation count and languages.
+        seed: What makes the bundle reproducible.
+        index: Whether to build the search index.
+        bundle_id: The manifest's ``bundle_id``.
+
+    Returns:
+        The manifest written.
+    """
+    root = Path(path)
+    synthetic_bundle(root, shape, seed=seed)
+    with Catalog(root, data_path=Embedded()) as cat:
+        with cat.bulk_load():
+            cat.put(prepare_sdmx(cat.registry))
+        if index:
+            cat.build_index()
+    producer = Producer(name="statcatlib.testing", version=metadata.version("statcatlib"))
+    return publish_bundle(root, bundle_id=bundle_id, producer=producer, product={"seed": seed})