atlas-python 0.12.0__tar.gz → 0.13.1__tar.gz

{atlas_python-0.12.0 → atlas_python-0.13.1}/.gitignore

@@ -33,3 +33,5 @@ atlas-python/python/atlas/_atlas.pyd
 atlas-python/python/atlas/_atlas.abi3.so
 atlas-python/python/atlas/_atlas*.so
 atlas-python/python/atlas/_atlas.pdb
+test-collection-netcdf-seadatanet/
+batch_to_atlas.py

{atlas_python-0.12.0 → atlas_python-0.13.1}/Cargo.lock

@@ -28,9 +28,9 @@ checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c"
 
 [[package]]
 name = "array-format"
-version = "0.11.0"
+version = "0.12.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f894cd59132aa6a00c360fef55469100c39da19c556bbf44a46bd647f7252fae"
+checksum = "f7379c3303a9b9f0693ccd040d85da31e302a601efeafae8a6cb69341e42de41"
 dependencies = [
  "bytes",
  "futures",
@@ -71,7 +71,7 @@ dependencies = [
 
 [[package]]
 name = "atlas-python"
-version = "0.12.0"
+version = "0.13.1"
 dependencies = [
  "atlas-rust",
  "ndarray",
@@ -86,7 +86,7 @@ dependencies = [
 
 [[package]]
 name = "atlas-rust"
-version = "0.12.0"
+version = "0.13.1"
 dependencies = [
  "array-format",
  "chrono",

{atlas_python-0.12.0 → atlas_python-0.13.1}/Cargo.toml

@@ -5,7 +5,7 @@ members = ["atlas-python"]
 # Registry name on crates.io (`atlas` is taken). The library is still
 # imported as `atlas` — see `[lib]` below.
 name = "atlas-rust"
-version = "0.12.0"
+version = "0.13.1"
 edition = "2024"
 description = "Directory-based store for thousands of N-dimensional datasets local or remote using object storage."
 license = "Apache-2.0"
@@ -22,7 +22,7 @@ all-features = true
 rustdoc-args = ["--cfg", "docsrs"]
 
 [dependencies]
-array-format = "0.11.0"
+array-format = "0.12.0"
 object_store = "0.13"
 serde = { version = "1", features = ["derive"] }
 serde_json = "1"

{atlas_python-0.12.0 → atlas_python-0.13.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: atlas-python
-Version: 0.12.0
+Version: 0.13.1
 Classifier: Development Status :: 3 - Alpha
 Classifier: Intended Audience :: Science/Research
 Classifier: Programming Language :: Python :: 3
@@ -220,6 +220,40 @@ for the full resolution order.
 - `bool` is available as an *attribute* type but not as an array dtype.
 - `binary`, `list[...]`, `fixed_size_list[...,N]` are reserved for a later release.
 
+## Cross-dataset pruning
+
+`ds.array_stats(name)` answers "what's the range of `temperature` in *this* dataset?" one at a
+time. To ask "which of my 10 000 datasets could contain a value above 25?" as a single
+vectorised scan, use the **pruning index** — a flattened, columnar view of the per-dataset
+statistics, one row per dataset:
+
+```python
+import numpy as np
+
+store = atlas.Atlas.open("/tmp/my_store")
+
+# Footer only — every column's collection-wide min/max, no column data fetched.
+summ = store.column_summaries()
+if summ["temperature"]["max"] < 25.0:
+    ...  # nothing can match — skip the column entirely
+
+# Fetch just the columns you need; each is a dict of numpy arrays over the row space.
+idx = store.pruning_index(arrays=["temperature"])
+col = idx["columns"]["temperature"]
+
+ok   = col["present"] & col["stats_valid"] & idx["live"]   # exclude gaps + deleted rows
+hits = np.where(ok & (col["max"] > 25.0))[0]
+candidates = [idx["datasets"][i] for i in hits]            # row -> dataset name
+```
+
+The return value is self-describing: `idx["datasets"][i]` names row `i`, `idx["live"]` is the
+delete mask (always `&` it in), and datasets that don't declare a column are explicit gaps
+(`present` is `False`, with `row_count` 0). Only the columns you ask for are fetched from
+storage, so the cost is independent of how many other columns the collection has. `min`/`max`
+keep their source type (strings come back as `bytes`, timestamps as int64 nanoseconds). See the
+[stats & scans guide](https://github.com/maris-development/atlas/blob/main/atlas-python/docs/guides/stats.md)
+for the full API, including `global_attrs=` / `array_attrs=` columns.
+
 ## Cloud / object storage
 
 With the `cloud` extra, `Atlas.open` / `Atlas.create` accept an
@@ -244,9 +278,13 @@ handle instead of a local path. The path-based local-filesystem API works withou
 | `add_xarray_dataset(ds, name, chunks=None, fill_value=None)` | Append an `xarray.Dataset` (does **not** flush). `fill_value` overrides the per-array fill (scalar or `{var: scalar}`); see [Missing data](#missing-data). |
 | `open_as_xarray_dataset(name) -> xr.Dataset` | Read a dataset back (chunked vars come back dask-backed). |
 | `open_as_many_xarray_dataset(names, concat_dim="dataset") -> xr.Dataset` | Open many datasets stacked along `concat_dim` (eager numpy). |
+| `pruning_index(arrays=None, global_attrs=None, array_attrs=None) -> dict` | Flattened statistics for **only** the requested columns; see [Cross-dataset pruning](#cross-dataset-pruning). |
+| `column_summaries() -> dict` | Every column's collection-wide min/max, read from the index footer alone (no column data fetched). |
+| `dataset_row(name) -> int \| None` | The dataset's fixed row ordinal in the pruning index. |
+| `row_slots() -> int` | Total row slots (live + tombstoned) — the pruning index's height. |
 | `flush()` | The single durability boundary — persist everything. |
 | `close()` | Alias for `flush()`; also the `with`-block exit. |
-| `compact()` | Reclaim tombstoned space across cached array files. |
+| `compact()` | Reclaim tombstoned space across cached array files, and renumber row ordinals. |
 | `__enter__` / `__exit__` | Context-manager support (`__exit__` calls `close()`). |
 
 ### `atlas.DatasetView`

{atlas_python-0.12.0 → atlas_python-0.13.1}/README.md

@@ -26,6 +26,7 @@ Each dataset is a named group of N-dimensional arrays with typed attributes —
 ```text
 my_store/
 ├── atlas.json               ← interned per-dataset schema + attribute-key namespace (JSON)
+├── pruning.idx              ← columnar min/max/count index across all datasets (see below)
 ├── _global/
 │   └── data.af         ← dataset-level (global) attribute values, one entry per dataset
 ├── temperature/
@@ -36,7 +37,7 @@ my_store/
     └── data.af
 ```
 
-`atlas.json` holds only the **schema** of the collection. Attribute **values** live in the `.af` files: per-variable attributes on the variable's own file, and dataset-level attributes in the reserved `_global` file.
+`atlas.json` holds only the **schema** of the collection. Attribute **values** live in the `.af` files: per-variable attributes on the variable's own file, and dataset-level attributes in the reserved `_global` file. `pruning.idx` is a derived, columnar statistics index — everything you need to prune a query to a handful of candidate datasets without opening a single `.af` file.
 
 ---
 
@@ -44,7 +45,7 @@ my_store/
 
 `atlas.json` is read **once** when the store is opened or created. Every subsequent mutation — `create_dataset`, `define_array`, `set_attribute`, `set_array_attribute`, `delete_array`, `delete_dataset` — only touches in-memory state; array writes buffer inside the per-array in-memory layer and attribute writes buffer in a pending-attribute map. **Nothing reaches disk until `Atlas::flush()` (or `Atlas::close()`).** Dropping an `Atlas` without flushing abandons every pending in-memory write.
 
-A single `Atlas::flush()` drains the buffered attributes into their `.af` files, walks every cached `ArrayFile` (writing deltas + stats), and then serialises the in-memory schema to `atlas.json`. This gives one durability boundary for the whole store: N datasets ⇒ one delta file per touched array name (not one per dataset) and one `atlas.json` rewrite (not N).
+A single `Atlas::flush()` drains the buffered attributes into their `.af` files, walks every cached `ArrayFile` (writing deltas + stats), fills and writes the `pruning.idx` statistics index, and then serialises the in-memory schema to `atlas.json`. This gives one durability boundary for the whole store: N datasets ⇒ one delta file per touched array name (not one per dataset) and one `atlas.json` rewrite (not N).
 
 `DatasetView` is a borrowed handle into the atlas's shared meta — it has no `flush()` of its own.
 
@@ -56,7 +57,7 @@ A single `Atlas::flush()` drains the buffered attributes into their `.af` files,
 
 The schema is a plain JSON file written on `Atlas::flush()` / `Atlas::close()`. It stores:
 
-- **Store version** — for format upgrades. The current version is `2`; stores written by an older atlas are rejected on open (re-export to upgrade).
+- **Store version** — for format upgrades. The current version is `3`; stores written by an older atlas are rejected on open with a clear `Error::UnsupportedVersion` (re-export to upgrade). Version 3 added the per-dataset **row ordinals + tombstone mask** that back the pruning index.
 - **Interned schema pool** — each *distinct* per-dataset schema is stored once and referenced by index. A dataset's schema is its array schemas (per array: dtype, shape, chunk shape, named dimensions, codec) plus its **attribute-key namespace** (the names of its global and per-variable attribute keys). A collection of thousands of homogeneous datasets stores its schema a single time.
 - **Dataset registry** — a map of dataset name → schema-pool index.
 - **Merged schema** — a collection-wide summary listing every unique array (with its dtype widened across all datasets, its dimensions, and its per-variable attribute types) and every global attribute type. Descriptive only — reads always use each dataset's own schema — but handy for tools that want one view of "what's in here." Also exposed programmatically via `Atlas::merged_schema()`.
@@ -86,6 +87,52 @@ Each array variable gets its own subdirectory with a single `data.af` binary fil
 - **Persisted statistics** — on `Atlas::flush()`, min, max, null count, and row count are computed per array per dataset and stored alongside the data. Cells equal to the array's fill value are tallied in `null_count` and excluded from `min`/`max` (NaN fills match NaN cells by bit pattern). Statistics survive store reopening.
 - **In-memory caches** — a 256 MiB decoded block cache and a 64 MiB raw I/O cache sit in front of the object store for repeated reads.
 
+### `pruning.idx`
+
+The per-array `.af` stats above answer "what's the min/max of `temperature` in *this* dataset?" one dataset at a time. `pruning.idx` answers the **cross-dataset** question — "which of my 10 000 datasets could possibly contain a value above 25?" — as a single vectorised scan, without opening any `.af` file.
+
+It is a flattened, **columnar** index, written at the store root on every `flush()`:
+
+- **One logical row per dataset**, positional — row *i* is the dataset at ordinal *i*. Deleted datasets keep their slot (see [Deletes](#deletes-and-compaction) below) and are hidden by a liveness mask, so a row number never changes underneath a cached index.
+- **One column per array, per global attribute, and per (array, attribute) pair.** Each column holds, for every dataset: a `present` bit, a `stats_valid` bit, `min`, `max`, `row_count`, and `null_count`. Datasets that don't declare that array/attribute are **explicit gaps** (`present = false`), so columns stay aligned no matter how heterogeneous the collection is.
+- **Compacted and column-addressable.** Sparsity is the norm (~26 % density on a real 10 000-dataset collection), so on disk each column stores values only for its present rows, keyed by the bitmap. Columns are compressed individually (Zstd by default, via `StoreConfig::pruning_compression`) and located through a footer directory — reading 2 of 200 columns fetches ~2/200ths of the file via ranged reads, never the whole thing.
+- **Guarded against staleness.** The footer carries a `meta_epoch` that must match `atlas.json`; an index left behind by a crash between the two writes is rejected (`Error::CorruptIndex`) rather than silently mis-attributing rows.
+
+The read API is a two-step funnel — rule columns out from the footer alone, then fetch only the survivors:
+
+```rust
+use atlas::{ColumnKey, StatVal};
+
+// Footer only — every column's collection-wide min/max, no column data fetched.
+for (key, summary) in store.column_summaries().await? {
+    // e.g. skip a column whose global max can't satisfy the predicate.
+    let _ = summary.might_match(|_, hi| hi > &StatVal::Float(25.0));
+}
+
+// Fetch just the columns you need. The returned index is self-describing:
+// it carries the liveness mask and the row↔name mapping.
+let key = ColumnKey::array("temperature");
+let index = store.pruning_index(std::slice::from_ref(&key)).await?;
+if let Some(view) = index.view(&key) {
+    // Deleted and absent rows are already excluded.
+    for row in view.candidates(|_, hi| hi > &StatVal::Float(25.0)) {
+        println!("candidate: {}", index.dataset_name(row).unwrap());
+    }
+}
+```
+
+`min`/`max` keep the type the statistic was computed with (string min/max are lexicographic bytes; timestamps are integer nanoseconds); a column's collection-wide *declared* type is the merged dtype from `atlas.json`. **Attribute columns currently record presence only** — which datasets carry the key — not attribute values; filtering on an attribute's *value* through the index isn't wired up yet.
+
+---
+
+## Deletes and compaction
+
+`delete_dataset` and `delete_array` are **logical**. A deleted dataset is *tombstoned* — its slot stays in `atlas.json` (so every later dataset keeps its row ordinal) and a liveness bit is cleared; its entries in the shared `.af` files are marked deleted but the bytes remain. Reads, `list_datasets`, the merged schema, and the pruning index all skip tombstoned datasets, so a delete is invisible to consumers immediately, but nothing is reclaimed yet.
+
+`Atlas::compact()` is the garbage collector. It rewrites every cached `.af` file with the tombstoned regions dropped, **renumbers** the surviving datasets to close the ordinal holes, and rebuilds the pruning index against the new numbering. It is the only operation that changes a dataset's row ordinal — so a row number cached from before a compact is stale afterwards.
+
+Re-creating a deleted name reuses its slot (and pruning-index row), starting from a clean schema — the revived dataset never inherits the previous occupant's stats.
+
 ---
 
 ## Quick start
@@ -149,8 +196,9 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
 | **Array** | An N-dimensional typed array with named dimensions and an optional chunk shape. |
 | **Attribute** | A typed scalar attached to a dataset (metadata, not array data). |
 | **Array file** | One `data.af` file per variable name, shared across all datasets that define that variable. |
-| **Flush** | `Atlas::flush()` — the single durability boundary. Persists every cached array file and rewrites `atlas.json` from the in-memory `StoreMeta`. Must be called explicitly (or via `Atlas::close()` / Python's `with atlas:`). |
-| **Compact** | `Atlas::compact()` rewrites every cached `.af` file to reclaim space after deletes. |
+| **Pruning index** | `pruning.idx` — a columnar min/max/count index across all datasets. Query it with `Atlas::pruning_index(&[ColumnKey])` / `column_summaries()` to prune to candidate datasets without reading array data. |
+| **Flush** | `Atlas::flush()` — the single durability boundary. Persists every cached array file, the pruning index, and rewrites `atlas.json` from the in-memory `StoreMeta`. Must be called explicitly (or via `Atlas::close()` / Python's `with atlas:`). |
+| **Compact** | `Atlas::compact()` rewrites every cached `.af` file to reclaim space after deletes, and renumbers dataset row ordinals. |
 | **StoreConfig** | Configuration passed to `Atlas::create`. Currently holds the compression `Codec`. |
 | **Codec** | Compression codec for new array blocks: `Codec::Zstd` (default), `Codec::Lz4`, or `Codec::Uncompressed`. Persisted in `atlas.json`; `open` reads it automatically. |
 
@@ -215,6 +263,7 @@ Reading `temperature` for 1 000 datasets means opening exactly **one file**. Thi
 | Cross-dataset column scan | Slow (N file opens) | Slow (N directory opens) | Fast (1 file open) |
 | Partial reads | Yes | Yes | Yes |
 | Statistics (min/max/nulls) | No | No | Yes (persisted on flush) |
+| Cross-dataset stat pruning | No | No | Yes (`pruning.idx`, vectorised) |
 | Self-describing metadata | Yes | Yes | Yes (`atlas.json`) |
 | Language support | C/Python/Julia/… | Python/Java/… | Rust |
 | Mutable after write | Limited | Yes | Yes (chunked overwrites + compact) |
@@ -262,10 +311,12 @@ Two caches sit in front of the object store:
 
 The decoded cache means repeated reads of the same chunk cost only a hash-map lookup. Both caches are shared across all `DatasetView`s that open the same `Atlas`.
 
-### Persisted statistics
+### Persisted statistics and pruning
 
 Min, max, null count, and row count are computed and persisted on every `Atlas::flush()`. Downstream systems can read these statistics from the opened `DatasetView` without touching array data at all — useful for query planning, dashboards, or data-quality checks.
 
+For **cross-dataset** query planning, the [`pruning.idx`](#pruningidx) index turns "which datasets could match?" into a vectorised scan over compacted, column-addressed statistics. A footer-only `column_summaries()` rules whole columns out for free; `pruning_index(&[…])` then fetches just the requested columns via ranged reads. On a real 10 000-dataset collection the whole index is ~0.3 MiB compressed, and a single-column query reads it in a couple of milliseconds — so a query engine can prune a fleet of datasets to a handful of candidates before opening a single `.af` file.
+
 ### Compression
 
 The codec is chosen once at store creation via `StoreConfig` and written into `atlas.json`. `Atlas::open` reads it from there, so callers never need to repeat the codec choice. Each array also records its own codec in `atlas.json`, so a store can theoretically hold arrays written with different codecs if the schema is migrated.
@@ -280,7 +331,7 @@ Choose LZ4 when decompression throughput matters more than storage size (e.g. la
 
 ### Write path
 
-Writes are buffered in-memory across the whole atlas. Calling `Atlas::flush()` compresses and writes every modified block across every cached array file, then rewrites `atlas.json` atomically (a single `PUT`). The write path scales with the number of modified chunks, not the number of datasets, and N consecutive `add_xarray_dataset` / `create_dataset` calls amortise to a single flush.
+Writes are buffered in-memory across the whole atlas. Attribute writes buffer in a shared, **interned** pending map — identical values (a NetCDF collection is ~94 % duplicate attribute values) are stored once and shared, so the buffer stays small during a large ingest. Calling `Atlas::flush()` compresses and writes every modified block across every cached array file, writes the pruning index, then rewrites `atlas.json` atomically (a single `PUT`). The write path scales with the number of modified chunks, not the number of datasets, and N consecutive `add_xarray_dataset` / `create_dataset` calls amortise to a single flush.
 
 ### Compaction
 

{atlas_python-0.12.0 → atlas_python-0.13.1}/atlas-python/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "atlas-python"
-version = "0.12.0"
+version = "0.13.1"
 edition = "2021"
 readme = "README.md"
 

{atlas_python-0.12.0 → atlas_python-0.13.1}/atlas-python/README.md

@@ -188,6 +188,40 @@ for the full resolution order.
 - `bool` is available as an *attribute* type but not as an array dtype.
 - `binary`, `list[...]`, `fixed_size_list[...,N]` are reserved for a later release.
 
+## Cross-dataset pruning
+
+`ds.array_stats(name)` answers "what's the range of `temperature` in *this* dataset?" one at a
+time. To ask "which of my 10 000 datasets could contain a value above 25?" as a single
+vectorised scan, use the **pruning index** — a flattened, columnar view of the per-dataset
+statistics, one row per dataset:
+
+```python
+import numpy as np
+
+store = atlas.Atlas.open("/tmp/my_store")
+
+# Footer only — every column's collection-wide min/max, no column data fetched.
+summ = store.column_summaries()
+if summ["temperature"]["max"] < 25.0:
+    ...  # nothing can match — skip the column entirely
+
+# Fetch just the columns you need; each is a dict of numpy arrays over the row space.
+idx = store.pruning_index(arrays=["temperature"])
+col = idx["columns"]["temperature"]
+
+ok   = col["present"] & col["stats_valid"] & idx["live"]   # exclude gaps + deleted rows
+hits = np.where(ok & (col["max"] > 25.0))[0]
+candidates = [idx["datasets"][i] for i in hits]            # row -> dataset name
+```
+
+The return value is self-describing: `idx["datasets"][i]` names row `i`, `idx["live"]` is the
+delete mask (always `&` it in), and datasets that don't declare a column are explicit gaps
+(`present` is `False`, with `row_count` 0). Only the columns you ask for are fetched from
+storage, so the cost is independent of how many other columns the collection has. `min`/`max`
+keep their source type (strings come back as `bytes`, timestamps as int64 nanoseconds). See the
+[stats & scans guide](https://github.com/maris-development/atlas/blob/main/atlas-python/docs/guides/stats.md)
+for the full API, including `global_attrs=` / `array_attrs=` columns.
+
 ## Cloud / object storage
 
 With the `cloud` extra, `Atlas.open` / `Atlas.create` accept an
@@ -212,9 +246,13 @@ handle instead of a local path. The path-based local-filesystem API works withou
 | `add_xarray_dataset(ds, name, chunks=None, fill_value=None)` | Append an `xarray.Dataset` (does **not** flush). `fill_value` overrides the per-array fill (scalar or `{var: scalar}`); see [Missing data](#missing-data). |
 | `open_as_xarray_dataset(name) -> xr.Dataset` | Read a dataset back (chunked vars come back dask-backed). |
 | `open_as_many_xarray_dataset(names, concat_dim="dataset") -> xr.Dataset` | Open many datasets stacked along `concat_dim` (eager numpy). |
+| `pruning_index(arrays=None, global_attrs=None, array_attrs=None) -> dict` | Flattened statistics for **only** the requested columns; see [Cross-dataset pruning](#cross-dataset-pruning). |
+| `column_summaries() -> dict` | Every column's collection-wide min/max, read from the index footer alone (no column data fetched). |
+| `dataset_row(name) -> int \| None` | The dataset's fixed row ordinal in the pruning index. |
+| `row_slots() -> int` | Total row slots (live + tombstoned) — the pruning index's height. |
 | `flush()` | The single durability boundary — persist everything. |
 | `close()` | Alias for `flush()`; also the `with`-block exit. |
-| `compact()` | Reclaim tombstoned space across cached array files. |
+| `compact()` | Reclaim tombstoned space across cached array files, and renumber row ordinals. |
 | `__enter__` / `__exit__` | Context-manager support (`__exit__` calls `close()`). |
 
 ### `atlas.DatasetView`

{atlas_python-0.12.0 → atlas_python-0.13.1}/atlas-python/docs/guides/datasets-and-arrays.md

@@ -56,6 +56,27 @@ jan = atlas.open_dataset("jan_2024")
 atlas.delete_dataset("feb_2024")
 ```
 
+## Deleting datasets
+
+`delete_dataset` is **logical**. The dataset immediately disappears from
+`list_datasets()`, reads, the merged schema, and the pruning index — but under
+the hood it is *tombstoned*: its slot stays in `atlas.json` and its bytes stay
+in the shared array files. This keeps every other dataset's **row ordinal
+stable** (`dataset_row(name)` never shifts under you, which is what lets a
+cached [pruning index](stats.md) stay valid across a delete).
+
+```python
+atlas.delete_dataset("feb_2024")
+atlas.dataset_exists("feb_2024")   # False — invisible immediately
+atlas.row_slots()                  # still counts the dead slot until compact
+```
+
+Call [`atlas.compact()`](durability.md) to actually reclaim the space: it
+rewrites the array files without the tombstoned regions and **renumbers** the
+surviving datasets to close the ordinal holes — the only operation that changes
+a row ordinal. Re-creating a deleted name reuses its slot with a clean schema
+(the revived dataset never inherits the old one's data or stats).
+
 ## Declaring an array
 
 `define_array` records the schema (dtype, dims, shape, chunking, fill

atlas_python-0.13.1/atlas-python/docs/guides/stats.md

@@ -0,0 +1,177 @@
+# Stats and scans
+
+Atlas computes and persists per-array summary statistics on every flush.
+You can scan thousands of datasets without reading any raw chunks.
+
+## What's tracked
+
+`view.array_stats(name)` returns a dict (or `None` if the array doesn't
+exist in this dataset, or hasn't been flushed yet):
+
+| Key | Meaning |
+|---|---|
+| `row_count` | The array's full logical size — the product of its shape. |
+| `null_count` | Cells equal to the `fill_value`, **plus every cell never written**. |
+| `min` | Minimum across written, non-fill cells. |
+| `max` | Maximum across written, non-fill cells. |
+
+`row_count` is capacity, not how much you wrote. An array declared with shape
+`[10]` reports `row_count == 10` even if you only ever wrote 5 cells — the
+other 5 show up in `null_count`, since they read back as the fill value:
+
+```python
+ds.define_array("t", dtype="int32", dims=["i"], shape=[10], chunk_shape=[5], fill_value=0)
+ds.write_array("t", start=[0], data=np.array([1, 2, 3, 4, 5], dtype=np.int32))
+atlas.flush()
+ds.array_stats("t")
+# {"row_count": 10, "null_count": 5, "min": 1, "max": 5}
+```
+
+So `row_count - null_count` is the count of real values.
+
+Stats are populated **after** [`atlas.flush()`](durability.md). Between a
+`define_array` / `write_array` and the next flush, `array_stats(name)`
+returns `None`.
+
+```python
+ds.write_array("readings", start=[0], data=values)
+ds.array_stats("readings")        # None — not flushed yet
+atlas.flush()
+ds.array_stats("readings")        # {"row_count": ..., "null_count": ..., "min": ..., "max": ...}
+```
+
+Stats live in a per-array `{array}/data.stats` sidecar, next to that array's
+`data.af` — **not** in `atlas.json`, which holds only schemas.
+
+## Cross-dataset scans: the pruning index
+
+`array_stats` answers for one array in one dataset. To ask "which datasets
+could possibly match this?" across a whole collection, use the **pruning
+index** — a flattened, columnar view with one row per dataset:
+
+```python
+store = atlas.Atlas.open("/tmp/store")
+
+idx = store.pruning_index(arrays=["readings"])
+col = idx["columns"]["readings"]
+
+# Vectorized: no Python loop over datasets.
+ok   = col["present"] & col["stats_valid"] & idx["live"]
+hits = np.where(ok & (col["max"] > 25.0))[0]
+candidates = [idx["datasets"][i] for i in hits]
+```
+
+Each column is a dict of numpy arrays, all the same length as `idx["rows"]`:
+
+| Key | Meaning |
+|---|---|
+| `present` | The dataset at this row declares the array/attribute. |
+| `stats_valid` | `min`/`max` are meaningful here. |
+| `min`, `max` | The per-dataset range. |
+| `row_count`, `null_count` | As above — **both 0** where `present` is `False`. |
+
+Row `i` is the dataset at ordinal `i` (`store.dataset_row(name)`), and
+`idx["datasets"][i]` maps back to its name. Datasets that don't declare the
+array are **explicit gaps** rather than missing entries, so columns stay
+aligned across the collection no matter how heterogeneous it is. A dataset
+without the array contributes `row_count == 0`, which is a real answer you can
+sum or filter on directly.
+
+`present` and `stats_valid` are separate on purpose: `stats_valid` is `False`
+where there is no usable range even though the dataset does declare the array
+— `List` dtypes, which carry no statistics at all, and rows written but not
+yet flushed. Their `row_count` / `null_count` are still valid, so check
+`present` for "does it have this?" and `stats_valid` before touching
+`min`/`max`.
+
+Deleted datasets keep their row and are masked by `idx["live"]`. Always `&` it
+in, or a deleted dataset's values will widen your result.
+
+### Read only what you need
+
+`pruning.idx` is column-addressed: asking for two columns fetches two byte
+ranges, not the file. On a 10 000-dataset collection with 781 columns, reading
+one column takes ~2 ms.
+
+Cheaper still, `column_summaries()` reads the **footer only** — every column's
+collection-wide min/max and present count, no column data at all:
+
+```python
+summaries = store.column_summaries()      # ~2 ms for 781 columns
+if summaries["readings"]["max"] < 25.0:
+    ...  # no dataset can match; skip fetching the column entirely
+```
+
+The index is compressed with zstd by default
+(`StoreConfig.pruning_compression`) — on that same 10 000-dataset collection
+it is 0.29 MB rather than 17.4 MB. Blocks are compressed individually, so
+single-column reads stay ranged whatever codec you pick, and the codec is
+recorded in the index itself so readers adapt without being told.
+
+### Types
+
+Statistics keep the type they were computed with — nothing is cast. `min` and
+`max` come back as `int64`, `uint64`, `float64`, or a list of `bytes | None`,
+chosen from what the column actually holds. A column that is `int32` in some
+datasets and `float64` in others promotes to `float64` at the numpy boundary.
+
+For a column's collection-wide *declared* type, use
+[`merged_schema()`](datasets-and-arrays.md) — the index itself stores no dtype.
+
+### Attribute columns
+
+Attribute columns currently record **presence only**: `present` tells you which
+datasets carry the key, but `stats_valid` is `False` and `min`/`max` are unset.
+Filtering on attribute *values* through the index isn't wired up yet — read
+them per dataset with `attributes()` for now.
+
+[`examples/06_stats_scan.py`](https://github.com/maris-development/atlas/blob/main/atlas-python/examples/06_stats_scan.py)
+runs a per-dataset scan across 32 sensor datasets.
+
+## How `null_count` works with `fill_value`
+
+A "null" in atlas is **a cell whose stored value equals the array's
+`fill_value`**. Both *unwritten* cells (which read back as the fill value)
+and *written* cells whose value happens to match contribute to the count.
+
+For float arrays the natural choice is `fill_value=float("nan")` — NaN
+doesn't compare equal to itself by IEEE rules, but atlas special-cases
+NaN, so a written NaN counts as a null.
+
+```python
+ds.define_array("temp", dtype="float32", ..., fill_value=float("nan"))
+ds.write_array("temp", start=[0], data=np.array([1.0, np.nan, 3.0], dtype=np.float32))
+atlas.flush()
+ds.array_stats("temp")
+# {"row_count": 3, "null_count": 1, "min": 1.0, "max": 3.0}
+```
+
+For integer arrays, pick a sentinel value (`-1`, `np.iinfo(dtype).min`,
+etc.) — any *written* cell equal to it will also be counted as null. Pick
+a value that can't appear in real data.
+
+When ingesting via [`add_xarray_dataset`](xarray.md#fill-values-and-missing-data)
+you don't set these by hand: float arrays default to a `NaN` fill, datetimes
+to `NaT`, and strings to `""`, so cells masked by `mask_and_scale=True` are
+counted as null automatically. Pass `fill_value=` to override.
+
+## What stats don't include
+
+- **No mean, sum, stdev, or quantiles.** If you need those, read the array
+  and compute them with numpy / dask. Atlas's stats are designed to be
+  cheap-on-write and cheap-on-read, not a full analytics engine.
+- **No per-dimension reductions.** `min` / `max` are scalars over the
+  whole array, not per-row / per-column.
+- **No per-chunk zone maps.** Statistics are per array per dataset; there is
+  no finer granularity to prune within a large array.
+- **`List` / `FixedSizeList` arrays get no statistics at all** — not even
+  counts.
+
+Strings and timestamps *are* covered. String `min`/`max` come back as `bytes`
+and order lexicographically; timestamps come back as integer nanoseconds since
+the epoch:
+
+```python
+ds.array_stats("station")["min"]    # b"AAA01"
+np.array(ds.array_stats("time")["min"]).astype("datetime64[ns]")
+```

{atlas_python-0.12.0 → atlas_python-0.13.1}/atlas-python/docs/index.md

@@ -40,6 +40,11 @@ with atlas.Atlas.create("/tmp/my_store", codec="zstd") as store:
 - **Bulk cross-dataset APIs** — `open_as_many_xarray_dataset` and
   `read_array_across_stacked` collapse N per-dataset reads into one PyO3
   call, sharing a single file handle.
+- **Cross-dataset pruning** — a columnar min/max/count index
+  (`store.pruning_index(...)` / `column_summaries()`) prunes "which of my
+  10 000 datasets could match this predicate?" to a handful of candidates
+  with a single vectorised scan, without opening any array file. See
+  [Stats and scans](guides/stats.md).
 - **Sync API, GIL-released** — a multi-threaded tokio runtime backs every
   blocking call; the GIL is released so other Python threads can run.
 - **Local fs or cloud storage** — pass a path string for local, or an

{atlas_python-0.12.0 → atlas_python-0.13.1}/atlas-python/src/attr.rs

@@ -8,19 +8,19 @@ pub fn py_to_attr(value: &Bound<'_, PyAny>, dtype_hint: Option<&str>) -> PyResul
         return py_to_attr_typed(value, hint);
     }
 
-    if let Ok(b) = value.downcast::<PyBool>() {
+    if let Ok(b) = value.cast::<PyBool>() {
         return Ok(Attr::Bool(b.is_true()));
     }
-    if value.downcast::<PyString>().is_ok() {
+    if value.cast::<PyString>().is_ok() {
         return Ok(Attr::String(value.extract::<String>()?));
     }
-    if value.downcast::<PyBytes>().is_ok() {
+    if value.cast::<PyBytes>().is_ok() {
         return Ok(Attr::Binary(value.extract::<Vec<u8>>()?));
     }
-    if value.downcast::<PyInt>().is_ok() {
+    if value.cast::<PyInt>().is_ok() {
         return Ok(Attr::Int64(value.extract::<i64>()?));
     }
-    if value.downcast::<PyFloat>().is_ok() {
+    if value.cast::<PyFloat>().is_ok() {
         return Ok(Attr::Float64(value.extract::<f64>()?));
     }
     Err(PyValueError::new_err(format!(