statcatlib 0.13.0__tar.gz → 0.15.0__tar.gz

{statcatlib-0.13.0 → statcatlib-0.15.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: statcatlib
-Version: 0.13.0
+Version: 0.15.0
 Summary: Queryable open data catalog engine for DCAT-AP, StatDCAT-AP, CKAN, and SDMX
 Keywords: sdmx,dcat,open-data,catalog,duckdb,statistical-data,ckan
 Author: gabrielgellner
@@ -18,7 +18,11 @@ Classifier: Topic :: Internet :: WWW/HTTP :: Indexing/Search
 Requires-Dist: attrs>=26.1.0
 Requires-Dist: duckdb>=2.0.0
 Requires-Dist: httpx>=0.28
-Requires-Dist: sdmxlib>=0.84,<0.85
+Requires-Dist: sdmxlib>=0.86,<0.87
+Requires-Dist: obstore>=0.12,<0.13 ; extra == 'blob'
+Requires-Dist: azure-identity>=1 ; extra == 'blob'
+Requires-Dist: typer>=0.15 ; extra == 'blob'
+Requires-Dist: rich>=15.0 ; extra == 'blob'
 Requires-Dist: lancedb>=0.39 ; extra == 'search'
 Requires-Dist: sentence-transformers>=3.0 ; extra == 'search'
 Requires-Dist: typer>=0.15 ; extra == 'search'
@@ -29,6 +33,7 @@ Project-URL: Repository, https://github.com/StatCat-Suite/statcatlib
 Project-URL: Documentation, https://statcat-suite.github.io/statcatlib
 Project-URL: Issue Tracker, https://github.com/StatCat-Suite/statcatlib/issues
 Project-URL: Changelog, https://github.com/StatCat-Suite/statcatlib/blob/main/CHANGELOG.md
+Provides-Extra: blob
 Provides-Extra: search
 Description-Content-Type: text/markdown
 
@@ -462,9 +467,10 @@ just hooks-install
 ```
 
 ```bash
-just ci                # full gate: index guard + lint + format + docstrings + prose + Any ratchet + pyrefly + unit and integration tests + strict docs build
+just ci                # full gate: index guard + lint + format + docstrings + prose + Any ratchet + pyrefly + unit and integration tests + stores tier + strict docs build
 just test              # unit tests
 just test-integration  # integration tests (no network)
+just test-stores       # the blob writer, the upload and remote catalogs against Azurite (needs a container runtime)
 just test-live         # live tests against real SDMX endpoints
 just lint              # ruff check
 just typecheck         # pyrefly

{statcatlib-0.13.0 → statcatlib-0.15.0}/README.md

@@ -428,9 +428,10 @@ just hooks-install
 ```
 
 ```bash
-just ci                # full gate: index guard + lint + format + docstrings + prose + Any ratchet + pyrefly + unit and integration tests + strict docs build
+just ci                # full gate: index guard + lint + format + docstrings + prose + Any ratchet + pyrefly + unit and integration tests + stores tier + strict docs build
 just test              # unit tests
 just test-integration  # integration tests (no network)
+just test-stores       # the blob writer, the upload and remote catalogs against Azurite (needs a container runtime)
 just test-live         # live tests against real SDMX endpoints
 just lint              # ruff check
 just typecheck         # pyrefly

{statcatlib-0.13.0 → statcatlib-0.15.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "statcatlib"
-version = "0.13.0"
+version = "0.15.0"
 description = "Queryable open data catalog engine for DCAT-AP, StatDCAT-AP, CKAN, and SDMX"
 license = "Apache-2.0"
 license-files = ["LICENSE"]
@@ -29,7 +29,7 @@ dependencies = [
     "attrs>=26.1.0",
     "duckdb>=2.0.0",
     "httpx>=0.28",
-    "sdmxlib>=0.84,<0.85",
+    "sdmxlib>=0.86,<0.87",
 ]
 
 [[project.authors]]
@@ -43,6 +43,12 @@ search = [
     "typer>=0.15",
     "rich>=15.0",
 ]
+blob = [
+    "obstore>=0.12,<0.13",
+    "azure-identity>=1",
+    "typer>=0.15",
+    "rich>=15.0",
+]
 
 [project.scripts]
 statcat = "statcatlib.cli:main"
@@ -77,6 +83,7 @@ markers = [
     "unit: fast, in-process tests with no network I/O",
     "integration: tests that use real DuckDB and sdmxlib but no network",
     "live: tests that make real HTTP calls to external SDMX endpoints",
+    "stores: tests that run against an object-store emulator in a container",
 ]
 
 [tool.coverage.run]
@@ -95,9 +102,11 @@ exclude_lines = [
 
 [dependency-groups]
 dev = [
+    "azure-identity>=1",
     "lancedb>=0.39",
     "mkdocs-material>=9.7.6",
     "mkdocstrings[python]>=1.0.3",
+    "obstore>=0.12,<0.13",
     "polars>=1.39.3",
     "pydoclint>=0.11.1,<0.12",
     "pyrefly>=1.4.0,<1.5",

{statcatlib-0.13.0 → statcatlib-0.15.0}/pyproject.toml.orig

@@ -1,6 +1,6 @@
 [project]
 name = "statcatlib"
-version = "0.13.0"
+version = "0.15.0"
 description = "Queryable open data catalog engine for DCAT-AP, StatDCAT-AP, CKAN, and SDMX"
 license = "Apache-2.0"
 license-files = ["LICENSE"]
@@ -27,13 +27,14 @@ dependencies = [
     # by DuckDB 1.x, so by statcatlib 0.8 or earlier.
     "duckdb>=2.0.0",
     "httpx>=0.28",
-    # <0.85: every suite library is pre-1.0, so a minor bump may break
+    # <0.87: every suite library is pre-1.0, so a minor bump may break
     # (GH-67).
-    "sdmxlib>=0.84,<0.85",
+    "sdmxlib>=0.86,<0.87",
 ]
 
 [project.optional-dependencies]
 search = ["lancedb>=0.39", "sentence-transformers>=3.0", "typer>=0.15", "rich>=15.0"]
+blob = ["obstore>=0.12,<0.13", "azure-identity>=1", "typer>=0.15", "rich>=15.0"]
 
 [project.scripts]
 statcat = "statcatlib.cli:main"
@@ -83,6 +84,7 @@ markers = [
     "unit: fast, in-process tests with no network I/O",
     "integration: tests that use real DuckDB and sdmxlib but no network",
     "live: tests that make real HTTP calls to external SDMX endpoints",
+    "stores: tests that run against an object-store emulator in a container",
 ]
 
 [tool.coverage.run]
@@ -104,9 +106,11 @@ exclude_lines = [
 
 [dependency-groups]
 dev = [
+    "azure-identity>=1",
     "lancedb>=0.39",
     "mkdocs-material>=9.7.6",
     "mkdocstrings[python]>=1.0.3",
+    "obstore>=0.12,<0.13",
     "polars>=1.39.3",
     # The checkers are capped, as sdmxlib's are: uv.lock is gitignored, so an
     # uncapped checker's next release would change what `just ci` reports

{statcatlib-0.13.0 → statcatlib-0.15.0}/src/statcatlib/EGM.md

@@ -5,8 +5,9 @@
 `statcatlib` is the package root. `__init__.py` re-exports the public API of
 `model`, `sources` and `store`; `CatalogAdmin` from `admin.py`; `Catalog`, `Product`, `Edition`,
 `SealOutcomes` and `NoLiveEditionError` from `catalog.py`; the errors `InventoryConflictError`,
-`InventoryContentionError`, `InventoryRuleError`, `NonFastForwardError` and
-`CrossProductRuleError` from `store/inventory.py`, and `WorkingCopyExistsError`,
+`InventoryContentionError`, `InventoryRuleError`, `InventoryWriteRefusedError`,
+`NonFastForwardError` and `CrossProductRuleError` from `store/inventory.py`; `BlobLocation` and `TokenIdentity`
+from `store/blob.py`; `upload_edition`, `UploadReport` and `UploadCheckError` from `upload.py`; `WorkingCopyExistsError`,
 `WorkingCopyClosedError`, `SealCheckError`, `EditionCheckError` and `JournalError` from
 `store/working_copy.py`; `NotLoadedError` and `is_unloaded` from the store's
 sentinel module; and part of `expr` (listed below). It exposes `checks`,
@@ -16,7 +17,7 @@ products (`catalog.py`), the operational changes to its Inventory
 (`admin.py`), seal checks (`checks.py`), the `statcat` section of
 an sdmxlib edition manifest (`bundle.py`), the integrity and support
 checks behind `statcat verify-catalog` and `statcat doctor`
-(`diagnostics.py`), composable query filters
+(`diagnostics.py`), the copy of a sealed edition to object storage (`upload.py`), composable query filters
 (`query.py`), traversal expressions (`expr.py`), SDMX URN builders (`urn.py`)
 and test catalogs (`testing.py`).
 
@@ -28,8 +29,8 @@ and test catalogs (`testing.py`).
   agents, distributions, SKOS concepts); imports only `InternationalString`
   from sdmxlib.
 - `store/`: [`./store/EGM.md`](./store/EGM.md) — the private `_Store`,
-  `EditionView`, `WorkingCopy` and the seal, the Inventory writer,
-  `QueryBuilder`, scopes, DDL, LanceDB search index.
+  `EditionView`, `WorkingCopy` and the seal, the Inventory writer and its
+  local and blob storage, `QueryBuilder`, scopes, DDL, LanceDB search index.
 - `sources/`: [`./sources/EGM.md`](./sources/EGM.md) — SDMX and CKAN ingest
   connectors.
 - `cli/`: [`./cli/EGM.md`](./cli/EGM.md) — the `statcat` Typer command.
@@ -45,7 +46,13 @@ and test catalogs (`testing.py`).
     or with `exist_ok=True` reopens it for writing and writes nothing.
   - `Catalog.open(path)`: read-only. Raises `FileNotFoundError` when *path*
     has no `catalog.json`. A catalog with no Inventory version has no
-    products.
+    products. A URI string raises `ValueError` naming `BlobLocation`.
+  - `Catalog.open(BlobLocation)`: the same over object storage, with a
+    `_Opening` whose `root` is `<cache_dir>/<catalog_id>`. No `cache_dir` raises
+    `ValueError`, and a `catalog_id` that cannot name a directory raises
+    `IncompatibleCatalogError`. `catalog_id` is read once; `refresh` and
+    `description` read through a `BlobInventoryStorage`, and `repr` and messages
+    show `location.uri`. `Catalog.create` raises `TypeError` for one.
   - `description`: the `CatalogDescription` in the root's
     `extensions.statcat` of the Inventory version the live view holds (as of
     the last `refresh()`), read with `bundle.description_from_root`; `None`
@@ -94,8 +101,8 @@ and test catalogs (`testing.py`).
   `Edition`, all recorded in one version) and `refused` (product id to the
   error: `SealCheckError`, `NonFastForwardError` or another
   `InventoryRuleError` (`CrossProductRuleError` from the pre-check or the
-  record included), `ManifestError`, `EditionCheckError`, or
-  `InventoryContentionError`).
+  record included), `ManifestError`, `EditionCheckError`,
+  `InventoryContentionError`, or `InventoryWriteRefusedError`).
 - `Edition` is re-exported from `store/working_copy.py`.
 
 ### `admin.py`
@@ -202,6 +209,33 @@ The module keeps its 0.11 name and describes editions. Imported from
 - `STATCAT_SECTION = "statcat"`, `STATCAT_SECTION_FORMAT = 3`,
   `PRODUCT_KINDS = ("data", "structure")`.
 
+- `check_catalog_no_index(root, manifest, reg, profile) -> list[Finding]`:
+  `check_catalog` without the search index check, for an install with no
+  `lancedb`. `store/blob.py`'s `_statcat_check()` chooses between the two.
+
+### `upload.py`
+
+`statcat upload`'s work, on the `blob` extra. It reads and writes nothing under
+`catalog.json`, `inventory/` or `CURRENT`.
+
+- `upload_edition(edition_dir, location, *, release_at=None) -> UploadReport`:
+  refuses a store other than `AzureStore` (`TypeError`, naming W3) and a store with
+  no `TokenIdentity` (`ValueError`), then runs `_upload` with an `AzureBlobRest`.
+- `_upload(edition_dir, location, *, release_at=None, rest)`: the steps, over any
+  store and any digest client with `head(path) -> RemoteFile | None`. Tests and
+  `record_uploaded` call it. A naive `release_at` raises `ValueError`.
+- `UploadReport(edition_id, product_id, sent, kept, registered)`: `sent` and `kept`
+  hold store paths, and `registered` is advisory, since a retried create can report
+  false against its own earlier success.
+- `UploadCheckError(findings)` (`RuntimeError`): any finding, an oversize file, a
+  path leaving its directory, a rotted local file, a differing manifest, or a
+  differing `pending/` document.
+- `_pending_document(edition_id, release_at) -> bytes`: the `pending/` document's
+  bytes, the contract `docs/catalog-format.md` §1.5 states.
+- `_lookup(rest, manifest)`: adapts `rest.head` to sdmxlib's `Lookup`, asking once per
+  file. `_store_path(manifest, entry)` normalises `../data/`, which `obstore` refuses.
+- `MAX_PUT_BYTES`: Put Blob's limit, 5,000 MiB.
+
 ### `diagnostics.py`
 
 Imported from `statcatlib.diagnostics`; nothing here is re-exported from
@@ -211,7 +245,9 @@ read.
 - `verify_catalog(path) -> VerifyReport`: reads `FullInventory`; reports each
   root the read protocol skipped (`full.skipped`), each leaf that fails its
   hash or `validate_leaf`, and for every leaf entry a `manifest.json` whose
-  SHA-256 differs from the entry's, then runs
+  SHA-256 differs from the entry's, or, when it matches, each filled routing
+  field or artefact record `sdmxlib.inventory.verify_entry` finds differing,
+  with the leaf as the finding's file, then runs
   `check_edition(at="publish", structures=..., extensions={"statcat":
   check_catalog})`, with `structures=` from `_structures_for`: every recorded
   edition of every structure product for a data edition, the other
@@ -295,10 +331,29 @@ read.
   dataflow per table and the rows from `(seed, product_id)`, fills with
   `ingest_dataflows(..., partials=True)`, indexes `CL_GEO`'s codes with an
   `fts_only` `CodelistIndexPolicy` when lancedb is installed, and seals.
+  `codr_product` is `codr_working_copy(...).seal()`: `codr_working_copy`
+  takes the same arguments and returns the filled, indexed working copy,
+  unsealed. `codr_corpus(catalog, groups=(CorpusGroup(7000),), *, seed=0,
+  shape=..., embargoed=0.1, on_sealed=None) -> list[str]` seals each
+  `CorpusGroup(products, tables=1)`'s products, named by
+  `corpus_product_id` (`codr-<tables>t-<index:05d>`), through
+  `codr_product`, skipping products the Inventory records and discarding a
+  working copy a crash left; it seals the share `corpus_embargoed` picks (a
+  SplitMix64 draw per product id) between two `CatalogAdmin.set_embargo`
+  versions, turns the embargo off first and leaves it off, and returns the
+  products it sealed. `bench/corpus.py` (GH-124) drives it.
   `structure-classifications` holds `CL_CLASS_*`, `CL_UNIT` and the
   SDMX-agency `CL_OBS_STATUS`, so every codelist a table cuts has a
   structure home (A5). The draws are SplitMix64 seeded by SHA-256, so they
   do not depend on Python's `random`.
+- `inventory_corpus(writer, products=7000, *, embargoed=0.1, artefacts=False,
+  batch=500, seed=0) -> Root` writes `catalog.json` and one `try_commit` per
+  `batch` products, with no manifest behind them, so no rule runs. The leaves
+  come from sdmxlib's `record`, with hashed ids, one dataflow URN, `pull_bytes`
+  13,131,776 and, with `artefacts=True`, 10 artefact records. Commits start at
+  a fixed time and step one millisecond, so a seed writes the same bytes for a
+  given actor and statcatlib version. A catalog with a `catalog.json` raises
+  `FileExistsError`. `bench/inventory_read.py` (GH-98) drives it.
 
 ## Invariants
 
@@ -323,8 +378,14 @@ read.
   index and before it closes the store and registry; it then seals. The
   manifest records each file's digests, so a later write fails
   `check_edition`'s files check.
-- `Catalog.open` writes nothing. No `Catalog` or `Product` method takes or
-  returns a path inside the catalog.
+- `Catalog.open` writes nothing to the catalog. Over a `BlobLocation`, `Product.view`
+  writes to `cache_dir` alone: the live edition's files, by a rename of a verified,
+  fsynced staged directory, and the receive stamp. No embargoed edition's files and no
+  leaf bytes reach it. No `Catalog` or `Product` method takes or returns a path
+  inside the catalog.
+- A library call never takes a credential from the environment: `Catalog.open` over a
+  store takes a `BlobLocation` the caller built. `catalog.py` and `store/blob.py` read
+  no environment variable.
 - Only `CatalogAdmin` changes the Inventory, apart from a seal's record and
   the journal repair. No attribute of `Catalog`, `Product` or `EditionView`
   returns a `CatalogAdmin` or an embargoed, superseded or withdrawn edition.
@@ -336,7 +397,9 @@ read.
   `sdmxlib.LocalRegistry.edition`, which hashes the bytes of
   `manifest.json` it parses. A missing, malformed or different manifest
   raises sdmxlib's `ManifestError`, which `view` re-raises as
-  `IncompatibleCatalogError` naming `statcat verify-catalog`, chained. Every view a `Catalog` returns shares one
+  `IncompatibleCatalogError`, chained. A local catalog's message names
+  `statcat verify-catalog`, and a blob catalog's names the cached directory to
+  delete, so the edition is pulled again. Every view a `Catalog` returns shares one
   `sdmxlib.EditionEngine`, and `Catalog.close` closes them all.
 - A seal check only reads: `seal` runs every check against `wc.view()`
   before writing anything, and any finding raises `SealCheckError`.

{statcatlib-0.13.0 → statcatlib-0.15.0}/src/statcatlib/__init__.py

@@ -88,11 +88,13 @@ from statcatlib.store import (
     WorkingCopy,
     xhtml_to_markdown,
 )
+from statcatlib.store.blob import BlobLocation, TokenIdentity
 from statcatlib.store.inventory import (
     CrossProductRuleError,
     InventoryConflictError,
     InventoryContentionError,
     InventoryRuleError,
+    InventoryWriteRefusedError,
     NonFastForwardError,
 )
 from statcatlib.store.sentinel import NotLoadedError, is_unloaded
@@ -103,6 +105,7 @@ from statcatlib.store.working_copy import (
     WorkingCopyClosedError,
     WorkingCopyExistsError,
 )
+from statcatlib.upload import UploadCheckError, UploadReport, upload_edition
 
 __all__ = [
     "SEARCH_ANNOTATION_TYPES",
@@ -113,6 +116,7 @@ __all__ = [
     "AggregateDataset",
     "AliasedExpr",
     "BaseDataset",
+    "BlobLocation",
     "Catalog",
     "CatalogAdmin",
     "CatalogDescription",
@@ -159,6 +163,7 @@ __all__ = [
     "InventoryConflictError",
     "InventoryContentionError",
     "InventoryRuleError",
+    "InventoryWriteRefusedError",
     "JournalError",
     "LicenceDocument",
     "MicrodataDataset",
@@ -184,7 +189,10 @@ __all__ = [
     "StatisticalDataset",
     "TemporalCoverage",
     "ThemesScope",
+    "TokenIdentity",
     "TraversalExpr",
+    "UploadCheckError",
+    "UploadReport",
     "Variable",
     "WorkingCopy",
     "WorkingCopyClosedError",
@@ -201,6 +209,7 @@ __all__ = [
     "is_unloaded",
     "prepare_sdmx",
     "query",
+    "upload_edition",
     "urn",
     "xhtml_to_markdown",
 ]

{statcatlib-0.13.0 → statcatlib-0.15.0}/src/statcatlib/admin.py

@@ -362,8 +362,8 @@ class CatalogAdmin:
         (``rule="structure-home"``).
 
         An empty *reason* raises `ValueError`. It acts on the live edition at
-        this sdmxlib's store schema; rolling back another schema's live
-        edition arrives in Phase 4.
+        this sdmxlib's store schema, and does not roll back another schema's
+        live edition.
         """
         _require_reason(reason)
         if if_live is None:

{statcatlib-0.13.0 → statcatlib-0.15.0}/src/statcatlib/bundle.py

@@ -51,6 +51,7 @@ __all__ = [
     "STATCAT_SECTION_FORMAT",
     "catalog_section",
     "check_catalog",
+    "check_catalog_no_index",
     "description_from_json",
     "description_from_root",
     "description_to_json",
@@ -234,10 +235,19 @@ def _check_search(root: Path, declared: object) -> Iterator[Finding]:
         detail = f"{found}, which this build does not read: check it with that release, or rebuild with build_index()"
         yield Finding(STATCAT_SECTION, f"{_SEARCH_DIRNAME}/{_INDEX_MANIFEST}", detail)
         return
-    index = LanceIndex(search_dir)
+    try:
+        index = LanceIndex(search_dir)
+    except ImportError:
+        yield Finding(STATCAT_SECTION, _SEARCH_DIRNAME, "cannot check the search index without statcatlib[search]")
+        return
     if (declared_version := declared.get("index_version")) != index.version:
         detail = f"the manifest declares index_version {declared_version}, {_INDEX_MANIFEST} records {index.version}"
         yield Finding(STATCAT_SECTION, MANIFEST_NAME, detail)
+    yield from _check_layout(index, languages)
+
+
+def _check_layout(index: LanceIndex, languages: Mapping[str, object]) -> Iterator[Finding]:
+    """The index's tables: each declared language as declared, and none ``index.json`` does not name."""
     layout = index.describe()
     for lang in sorted(set(languages) | set(layout)):
         yield from _check_language(index, lang, layout.get(lang), languages.get(lang))
@@ -297,10 +307,36 @@ def check_catalog(root: Path, manifest: Manifest, reg: EditionRegistry, profile:
     Returns:
         Everything found wrong with the catalog's part of the edition.
     """
+    del profile
+    return _section_findings(root, manifest, reg, index=True)
+
+
+def check_catalog_no_index(root: Path, manifest: Manifest, reg: EditionRegistry, profile: Profile) -> list[Finding]:
+    """`check_catalog` without its search index check, for an install with no `lancedb`.
+
+    A ``statcatlib[blob]`` install holds no `lancedb`, so it cannot open the
+    index. The ``files`` check of `sdmxlib.edition.check_edition` still covers the
+    index's bytes by digest.
+
+    Args:
+        root: The edition directory.
+        manifest: Its manifest.
+        reg: sdmxlib's registry over the edition.
+        profile: The `check_edition` profile that runs it. Every profile runs
+            the same checks.
+
+    Returns:
+        Everything found wrong with the rest of the catalog's part of the edition.
+    """
+    del profile
+    return _section_findings(root, manifest, reg, index=False)
+
+
+def _section_findings(root: Path, manifest: Manifest, reg: EditionRegistry, *, index: bool) -> list[Finding]:
+    """The ``statcat`` checks, with the opening of the search index when *index* is true."""
     from statcatlib.store.schema import first_row  # noqa: PLC0415
     from statcatlib.store.store import annotation_fingerprint, statcat_schema_mismatch  # noqa: PLC0415
 
-    del profile
     conn = reg.connection
     rows = conn.execute(
         """
@@ -332,7 +368,9 @@ def check_catalog(root: Path, manifest: Manifest, reg: EditionRegistry, profile:
             for binding in manifest.bindings
         ]
         findings += structure_content(conn)
-    findings += [*_check_catalog_rows(conn, section), *_check_search(root, section.get("search"))]
+    findings += _check_catalog_rows(conn, section)
+    if index:
+        findings += _check_search(root, section.get("search"))
     search = section.get("search")
     if isinstance(search, Mapping):
         findings += _check_index_annotations(conn, search.get("annotations"), annotation_fingerprint)