atlas-python 0.9.0__tar.gz

atlas_python-0.9.0/.github/workflows/atlas-python-release.yaml

@@ -0,0 +1,115 @@
+name: atlas-python release
+
+on:
+    workflow_dispatch:
+    release:
+        types: [published]
+
+permissions:
+    contents: read
+
+jobs:
+    linux:
+        runs-on: ${{ matrix.runner }}
+        strategy:
+            fail-fast: false
+            matrix:
+                include:
+                    - target: x86_64
+                      runner: ubuntu-latest
+                    - target: aarch64
+                      runner: ubuntu-24.04-arm
+        steps:
+            - uses: actions/checkout@v4
+            - uses: actions/setup-python@v6
+              with:
+                  python-version: "3.10"
+            - name: Build wheels
+              uses: PyO3/maturin-action@v1
+              with:
+                  target: ${{ matrix.target }}
+                  manylinux: auto
+                  sccache: "true"
+                  args: --release --out dist --manifest-path atlas-python/Cargo.toml
+            - uses: actions/upload-artifact@v4
+              with:
+                  name: wheels-linux-${{ matrix.target }}
+                  path: dist
+
+    macos:
+        runs-on: macos-latest
+        strategy:
+            fail-fast: false
+            matrix:
+                target: [x86_64, aarch64]
+        steps:
+            - uses: actions/checkout@v4
+            - uses: actions/setup-python@v6
+              with:
+                  python-version: "3.10"
+            - name: Build wheels
+              uses: PyO3/maturin-action@v1
+              with:
+                  target: ${{ matrix.target }}
+                  sccache: "true"
+                  args: --release --out dist --manifest-path atlas-python/Cargo.toml
+            - uses: actions/upload-artifact@v4
+              with:
+                  name: wheels-macos-${{ matrix.target }}
+                  path: dist
+
+    windows:
+        runs-on: windows-latest
+        strategy:
+            fail-fast: false
+            matrix:
+                target: [x64]
+        steps:
+            - uses: actions/checkout@v4
+            - uses: actions/setup-python@v6
+              with:
+                  python-version: "3.10"
+                  architecture: ${{ matrix.target }}
+            - name: Build wheels
+              uses: PyO3/maturin-action@v1
+              with:
+                  target: ${{ matrix.target }}
+                  sccache: "true"
+                  args: --release --out dist --manifest-path atlas-python/Cargo.toml
+            - uses: actions/upload-artifact@v4
+              with:
+                  name: wheels-windows-${{ matrix.target }}
+                  path: dist
+
+    sdist:
+        runs-on: ubuntu-latest
+        steps:
+            - uses: actions/checkout@v4
+            - name: Build sdist
+              uses: PyO3/maturin-action@v1
+              with:
+                  command: sdist
+                  args: --out dist --manifest-path atlas-python/Cargo.toml
+            - uses: actions/upload-artifact@v4
+              with:
+                  name: wheels-sdist
+                  path: dist
+
+    release:
+        name: Publish to PyPI
+        runs-on: ubuntu-latest
+        if: github.event_name == 'release'
+        needs: [linux, macos, windows, sdist]
+        permissions:
+            id-token: write
+        environment:
+            name: pypi
+            url: https://pypi.org/p/atlas-python
+        steps:
+            - uses: actions/download-artifact@v4
+              with:
+                  pattern: wheels-*
+                  path: dist
+                  merge-multiple: true
+            - name: Publish to PyPI
+              uses: pypa/gh-action-pypi-publish@release/v1

atlas_python-0.9.0/.github/workflows/atlas-rust-release.yaml

@@ -0,0 +1,53 @@
+name: atlas-rust release
+
+on:
+    workflow_dispatch:
+    release:
+        types: [published]
+
+permissions:
+    contents: read
+
+jobs:
+    test:
+        name: Test
+        runs-on: ubuntu-latest
+        steps:
+            - uses: actions/checkout@v4
+            - uses: dtolnay/rust-toolchain@stable
+            - uses: Swatinem/rust-cache@v2
+            - name: Build workspace
+              run: cargo build --workspace --verbose
+            - name: Test atlas crate
+              # atlas-python links `pyo3/extension-module` (no libpython), so a
+              # plain `cargo test` binary can't link there — test the core crate.
+              run: cargo test -p atlas-rust --verbose
+
+    publish:
+        name: Publish to crates.io
+        runs-on: ubuntu-latest
+        needs: test
+        steps:
+            - uses: actions/checkout@v4
+            - uses: dtolnay/rust-toolchain@stable
+            - uses: Swatinem/rust-cache@v2
+
+            - name: Verify tag matches crate version
+              if: github.event_name == 'release'
+              run: |
+                  crate_version="$(cargo metadata --no-deps --format-version 1 \
+                      | python3 -c "import json,sys; print(next(p['version'] for p in json.load(sys.stdin)['packages'] if p['name']=='atlas-rust'))")"
+                  tag_version="${GITHUB_REF_NAME#v}"
+                  echo "crate=$crate_version tag=$tag_version"
+                  if [ "$crate_version" != "$tag_version" ]; then
+                      echo "::error::Tag $GITHUB_REF_NAME does not match crate version $crate_version"
+                      exit 1
+                  fi
+
+            - name: Package (dry run)
+              run: cargo publish -p atlas-rust --locked --dry-run
+
+            - name: Publish
+              env:
+                  CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }}
+              run: cargo publish -p atlas-rust --locked

atlas_python-0.9.0/.github/workflows/ci.yaml

@@ -0,0 +1,51 @@
+name: CI
+
+on:
+    push:
+        branches: [main]
+    pull_request:
+    workflow_dispatch:
+
+permissions:
+    contents: read
+
+concurrency:
+    group: ci-${{ github.ref }}
+    cancel-in-progress: true
+
+jobs:
+    rust:
+        name: Rust tests
+        runs-on: ubuntu-latest
+        steps:
+            - uses: actions/checkout@v4
+            - uses: dtolnay/rust-toolchain@stable
+            - uses: Swatinem/rust-cache@v2
+            - name: Build workspace
+              run: cargo build --workspace --verbose
+            - name: Test atlas crate
+              # atlas-python links `pyo3/extension-module` (no libpython), so a
+              # plain `cargo test` binary can't link there — test the core crate.
+              run: cargo test -p atlas-rust --verbose
+
+    python:
+        name: Python tests
+        runs-on: ubuntu-latest
+        strategy:
+            fail-fast: false
+            matrix:
+                python-version: ["3.10", "3.13"]
+        steps:
+            - uses: actions/checkout@v4
+            - uses: actions/setup-python@v6
+              with:
+                  python-version: ${{ matrix.python-version }}
+            - uses: dtolnay/rust-toolchain@stable
+            - uses: Swatinem/rust-cache@v2
+            - name: Build extension and run pytest
+              run: |
+                  python -m venv .venv
+                  source .venv/bin/activate
+                  python -m pip install --upgrade pip maturin
+                  maturin develop --extras test --manifest-path atlas-python/Cargo.toml
+                  pytest atlas-python/tests -v

atlas_python-0.9.0/.gitignore

@@ -0,0 +1,31 @@
+# Generated by Cargo
+# will have compiled files and executables
+debug
+target
+
+# These are backup files generated by rustfmt
+**/*.rs.bk
+
+# MSVC Windows builds of rustc generate these, which store debugging information
+*.pdb
+
+# Generated by cargo mutants
+# Contains mutation testing data
+**/mutants.out*/
+
+# RustRover
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+#.idea/
+.venv/
+__pycache__/
+.pytest_cache/
+raw_smoke_tests/
+.DS_Store
+.claude/
+pyatlas/python/pyatlas/_pyatlas.pyd
+pyatlas/python/pyatlas/_pyatlas.abi3.so
+playground/
+data/

atlas_python-0.9.0/CONTRIBUTING.md

@@ -0,0 +1,218 @@
+# Contributing to ATLAS
+
+Thanks for your interest. This document walks through what the project is, how the pieces fit, and how to set up a development environment.
+
+For end-user docs see the top-level [README.md](README.md) (Rust crate) and [pyatlas/README.md](pyatlas/README.md) (Python bindings).
+
+---
+
+## What this is
+
+**ATLAS** (Aggregated Tensor Large Array Store) is a directory-based store for many similarly-shaped, named, N-dimensional arrays. The design goal is fast cross-dataset variable scans — opening a single file to read variable `X` across N datasets, rather than N files.
+
+The repository is a Cargo workspace containing two crates:
+
+| Crate | Purpose |
+| --- | --- |
+| `atlas` (workspace root) | Rust library implementing the store and on-disk format. |
+| `pyatlas` ([pyatlas/](pyatlas/)) | PyO3 Python bindings + xarray integration. |
+
+---
+
+## How it works (architecture)
+
+### On-disk layout
+
+```
+my_store/
+├── atlas.json               ← dataset registry + per-dataset attributes (JSON)
+├── temperature/
+│   └── data.af              ← ArrayFile: every dataset's "temperature" in one file
+├── pressure/
+│   └── data.af
+└── time/
+    └── data.af
+```
+
+Every variable name owns one physical `.af` file shared by all datasets that define it (variable-first). The `.af` format comes from the [`array-format`](https://github.com/robinskil/array-format) crate — it's a columnar, chunk-oriented binary container with per-block compression and persisted statistics.
+
+`atlas.json` is the catalog. It records:
+- Store version, store-level codec.
+- Each dataset: its array schemas (dtype, shape, chunk shape, dimension names) and typed attributes.
+
+### Rust crate (`atlas`)
+
+| File | Role |
+| --- | --- |
+| [src/lib.rs](src/lib.rs) | Public re-exports; `validate_name`; thread-safety asserts. |
+| [src/store.rs](src/store.rs) | The `Atlas` struct: open/create, dataset CRUD, path-based wrappers (`open_path` / `create_path`). |
+| [src/dataset.rs](src/dataset.rs) | The `DatasetView` struct: define/read/write arrays, attributes, flush, compact. Also the shared `ArrayFile` cache. |
+| [src/meta.rs](src/meta.rs) | `atlas.json` (de)serialisation. |
+| [src/schema.rs](src/schema.rs) | `ArraySchema` and the typed `Attr` enum. |
+| [src/config.rs](src/config.rs) | `StoreConfig`, `Codec` (`Zstd` / `Lz4` / `Uncompressed`). |
+| [src/error.rs](src/error.rs) | `Error` / `Result`. |
+
+The API is async (tokio). Each physical array file is guarded by a `tokio::sync::RwLock` — reads share, writes are exclusive. The cache map uses `parking_lot::RwLock` and is never held across an `await`.
+
+### Python bindings (`pyatlas`)
+
+Mixed Python/Rust maturin layout:
+
+```
+pyatlas/
+├── Cargo.toml               ← cdylib named `_pyatlas`
+├── pyproject.toml           ← maturin build backend
+├── python/pyatlas/
+│   ├── __init__.py          ← re-exports + xarray accessor registration
+│   ├── __init__.pyi         ← type stubs (PEP 561)
+│   ├── py.typed             ← marker
+│   └── xarray.py            ← xarray integration + ds.atlas accessor
+├── src/
+│   ├── lib.rs               ← #[pymodule] _pyatlas wiring
+│   ├── runtime.rs           ← shared OnceLock<tokio::Runtime>
+│   ├── error.rs             ← atlas::Error → PyErr mapping
+│   ├── dtype.rs             ← dtype string parsing & DType ↔ name
+│   ├── attr.rs              ← Attr ↔ Python primitive conversion
+│   ├── store.rs             ← PyAtlas (wraps atlas::Atlas)
+│   └── dataset.rs           ← PyDatasetView (wraps atlas::DatasetView)
+├── tests/                   ← pytest tests
+└── examples/                ← runnable example scripts
+```
+
+Key design points:
+- **Sync Python API, backed by an internal multi-threaded tokio runtime.** Each blocking call uses `py.allow_threads(|| RT.block_on(...))` so other Python threads can run.
+- **Type dispatch via macros.** Atlas's read/write methods are generic over dtype (`T: ArrayElement`); the bindings dispatch at runtime via a `numeric_dispatch!` macro in [pyatlas/src/dataset.rs](pyatlas/src/dataset.rs).
+- **`numpy`-zero-copy on the numeric path.** Python `np.ndarray` ↔ `ndarray::ArrayView` via the `numpy` crate.
+- **xarray integration** lives in [pyatlas/python/pyatlas/xarray.py](pyatlas/python/pyatlas/xarray.py). The `Atlas.add_xr_dataset` Rust pymethod and the `ds.atlas.write` accessor both delegate to the `_write_xarray_new_dataset` helper. dask-backed variables are streamed one chunk at a time via `arr.blocks[idx].compute()`.
+
+---
+
+## Prerequisites
+
+| Tool | Why |
+| --- | --- |
+| **Rust** (stable, 1.85+ for edition 2024 in the atlas crate) | Build the workspace. |
+| **Python ≥ 3.9** | Run / build pyatlas (wheel targets `abi3-py39`). |
+| **`maturin`** | Build the Python extension. `pip install maturin`. |
+
+Install Rust via [rustup.rs](https://rustup.rs). Python 3.13 is what the maintainers develop against.
+
+---
+
+## Building & testing the Rust crate
+
+```bash
+cargo build                       # compile the atlas crate + workspace
+cargo test                        # run all tests (60 in atlas, plus pyatlas check-only)
+cargo test -p atlas               # just the atlas crate (50 unit + 10 integration)
+cargo run --example lifecycle     # try one of the Rust examples
+```
+
+Examples live in [examples/](examples/) — `lifecycle.rs`, `sensor_fleet.rs`, `weather_store.rs`.
+
+---
+
+## Building & installing the Python library
+
+The Python extension links against libpython, so plain `cargo build -p pyatlas` will fail at link time. Use `maturin develop` (which sets the right flags and installs into the active virtualenv).
+
+### One-time setup
+
+```bash
+# From the repo root
+python3.13 -m venv .venv
+source .venv/bin/activate
+pip install --upgrade pip maturin
+```
+
+`pyatlas` already lists `numpy`, `xarray`, and `dask` as required runtime deps, so `maturin develop` will pull them in automatically.
+
+### Development build
+
+```bash
+cd pyatlas
+maturin develop --release         # builds, then editable-installs into the active venv
+```
+
+After `maturin develop`, the package is editable: changes to `python/pyatlas/*.py` take effect immediately, but Rust changes require re-running `maturin develop`.
+
+### Building a distributable wheel
+
+```bash
+cd pyatlas
+maturin build --release           # produces pyatlas-0.1.0-*.whl in target/wheels/
+```
+
+### Verification
+
+```bash
+.venv/bin/python -c "import pyatlas; print(pyatlas.__version__)"
+```
+
+---
+
+## Running the Python tests
+
+```bash
+.venv/bin/pip install pytest      # one-time
+.venv/bin/pytest pyatlas/tests/ -v
+```
+
+The two test files:
+- [pyatlas/tests/test_smoke.py](pyatlas/tests/test_smoke.py) — exercises the core pyatlas API.
+- [pyatlas/tests/test_xarray.py](pyatlas/tests/test_xarray.py) — xarray accessor + dask streaming.
+
+After every Rust change re-run `maturin develop` first, otherwise pytest will run against the previously-built binary.
+
+---
+
+## Running the Python examples
+
+Self-contained scripts under [pyatlas/examples/](pyatlas/examples/) that write to temp directories:
+
+```bash
+.venv/bin/python pyatlas/examples/01_basics.py
+.venv/bin/python pyatlas/examples/02_xarray.py
+.venv/bin/python pyatlas/examples/03_dask_streaming.py
+```
+
+---
+
+## Common dev workflows
+
+### Adding a new array dtype
+
+1. Add the variant to `atlas::DType` (depends on `array-format` supporting it).
+2. Add the dispatch arm in [pyatlas/src/dataset.rs](pyatlas/src/dataset.rs)'s `numeric_dispatch!` (or the explicit String/Binary branches).
+3. Extend [pyatlas/src/dtype.rs](pyatlas/src/dtype.rs) to parse the new dtype name.
+4. Update [pyatlas/python/pyatlas/xarray.py](pyatlas/python/pyatlas/xarray.py)'s `_NUMPY_TO_ATLAS` map if it has a numpy equivalent.
+5. Add a smoke test in [pyatlas/tests/test_smoke.py](pyatlas/tests/test_smoke.py).
+
+### Exposing a new `DatasetView` / `Atlas` method to Python
+
+1. Implement on the Rust side in [src/store.rs](src/store.rs) or [src/dataset.rs](src/dataset.rs).
+2. Wrap as a pymethod in [pyatlas/src/store.rs](pyatlas/src/store.rs) / [pyatlas/src/dataset.rs](pyatlas/src/dataset.rs). Use `py.allow_threads(|| runtime().block_on(...))` for async calls.
+3. Add the stub in [pyatlas/python/pyatlas/\_\_init\_\_.pyi](pyatlas/python/pyatlas/__init__.pyi).
+4. `cd pyatlas && maturin develop --release`.
+5. Write a test.
+
+### Touching the on-disk format
+
+Format changes are breaking — bump the `version` field in `StoreMeta` ([src/meta.rs](src/meta.rs)) and load both versions during `load_meta` until a migration path is clear.
+
+---
+
+## Code style
+
+- Rust: `cargo fmt` before committing; we follow the default rustfmt configuration. `cargo clippy --all-targets` should be clean.
+- Python: top-level scripts and tests use ordinary 4-space indentation and import ordering (stdlib, third-party, local). No formatter is enforced; match the existing style.
+- Comments and docstrings explain *why*, not what — the code already shows what.
+
+---
+
+## Pull requests
+
+1. Branch from `main`.
+2. Make sure `cargo test`, `cargo clippy --all-targets`, and `pytest pyatlas/tests/` all pass.
+3. If you change the on-disk format, the public API, or any behaviour visible to end users, update the relevant README.
+4. Keep commits focused; squash trivial fixups before review.