apairo 0.2.0__tar.gz

apairo-0.2.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 Augustin BRESSET
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

apairo-0.2.0/PKG-INFO

@@ -0,0 +1,286 @@
+Metadata-Version: 2.4
+Name: apairo
+Version: 0.2.0
+Summary: Extensible framework to load robotics datasets
+Author-email: Augustin Bresset <augustin.bresset@gmail.com>
+License: MIT
+Project-URL: Homepage, https://github.com/apairo-robotics/apairo
+Project-URL: Repository, https://github.com/apairo-robotics/apairo
+Project-URL: Issues, https://github.com/apairo-robotics/apairo/issues
+Keywords: lidar,robotics,dataset,point-cloud,traversability
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Classifier: Topic :: Scientific/Engineering :: Information Analysis
+Classifier: Intended Audience :: Science/Research
+Requires-Python: >=3.11
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: numpy
+Requires-Dist: PyYAML
+Provides-Extra: torch
+Requires-Dist: torch; extra == "torch"
+Provides-Extra: vision
+Requires-Dist: Pillow; extra == "vision"
+Provides-Extra: mnt
+Requires-Dist: zarr>=3.0; extra == "mnt"
+Requires-Dist: Pillow; extra == "mnt"
+Provides-Extra: dev
+Requires-Dist: pytest; extra == "dev"
+Requires-Dist: Pillow; extra == "dev"
+Requires-Dist: zarr>=3.0; extra == "dev"
+Provides-Extra: docs
+Requires-Dist: mkdocs>=1.6; extra == "docs"
+Requires-Dist: mkdocs-material>=9.5; extra == "docs"
+Requires-Dist: mkdocstrings[python]>=0.25; extra == "docs"
+Dynamic: license-file
+
+# apairo
+
+Unified Python loader for robotics sensor datasets — one API across synchronous and asynchronous layouts, with built-in preprocessing, filtering, and dataset composition.
+
+All data is returned as `numpy.ndarray`. Convert to the framework of your choice.
+
+---
+
+## Installation
+
+```bash
+pip install apairo
+```
+
+Optional extras:
+
+```bash
+pip install apairo[torch]    # PyTorch support (.pt files)
+pip install apairo[vision]   # Image loading (Pillow)
+```
+
+Requires Python ≥ 3.11.
+
+---
+
+## Quickstart
+
+```python
+import apairo
+
+ds = apairo.SemanticKittiDataset("/data/semantic_kitti", keys=["lidar", "labels"])
+sample = ds[0]
+# sample.data["lidar"]   -> np.ndarray (N, 4)  float32  [x, y, z, intensity]
+# sample.data["labels"]  -> np.ndarray (N,)    int64
+```
+
+---
+
+## Supported datasets
+
+| Class | Layout | Modalities |
+|---|---|---|
+| `SemanticKittiDataset` | synchronous | lidar, labels |
+| `Rellis3DDataset` | synchronous | lidar, labels, poses |
+| `Goose3DDataset` | synchronous | lidar, labels |
+| `MNTDataset` | synchronous | lidar, labels, poses |
+| `RawDataset` | asynchronous | any channels — declared in `.apairo/channels.yaml` |
+| `TartanKittiDataset` | asynchronous | any TartanDrive v2 channel |
+
+`RawDataset` is the profile-free loader for the asynchronous layout: it takes its
+channels — and their format (`npy`, `npys`, `bin`, `img`, `zarr`) — entirely from
+`.apairo/channels.yaml`, so it loads any such dataset, including the output of
+[apairo_extractor](https://github.com/apairo-robotics/apairo_extractor), with no
+code change.
+
+---
+
+## Command line
+
+Installing apairo provides the `apairo` command to inspect and initialize
+datasets from the terminal:
+
+```bash
+# Write/repair the .apairo sidecars by scanning a directory (root-aware, idempotent)
+apairo init /data/my_dataset
+
+# Show sequences, channels (tracked + untracked), event count and any issues
+apairo status /data/my_dataset           # add --json for machine output
+```
+
+`apairo init` reconstructs the `.apairo` files for data laid out before they
+existed (e.g. an older extraction) — no re-extraction needed — and the result
+loads directly with `RawDataset`. See [Command Line](https://apairo-robotics.github.io/apairo/cli/) for the full reference.
+
+---
+
+## Pipeline
+
+apairo provides a composable set of operations that chain together — each returns a full dataset:
+
+```python
+from apairo import Rellis3DDataset, FramePreprocessor
+from torch.utils.data import DataLoader
+import numpy as np
+
+# 1. Preprocess — run once, persisted in .apairo, reloaded transparently
+class TravLabel(FramePreprocessor):
+    output_key = "trav_gt";  output_loader = "npys"
+    input_keys = ["labels"]; timestamps_from = "lidar"; sources = ["labels"]
+    def process(self, sample): return (sample.data["labels"] < 10).astype(np.uint8)
+
+ds = Rellis3DDataset(root, keys=["lidar", "labels", "ground_height_csf"])
+ds.run_preprocess(TravLabel())
+
+# 2. Cache an expensive derived channel — computed once, served from RAM
+ds.transform("ground_height_csf", expensive_smooth)
+ds_prior = ds.select(["ground_height_csf"]).cache()
+
+# 3. Build train split — filter, join cached prior, apply augmentation
+valid = np.load("cache/valid_indices.npy")
+ds_train = (
+    Rellis3DDataset(root, keys=["lidar", "trav_gt"])
+    .filter(valid)
+    .join(ds_prior)
+    .transform("lidar", RangeFilter(max=50.0))
+)
+
+# 4. Drop into DataLoader — no adapter needed
+loader = DataLoader(ds_train, batch_size=8, shuffle=True, collate_fn=my_collate)
+```
+
+See [`examples/`](examples/) for complete runnable pipelines.
+
+---
+
+## Preprocessing
+
+Define a `FramePreprocessor` or `SequencePreprocessor`, run it once — apairo persists the output and reloads it transparently on subsequent runs.
+
+```python
+from apairo.preprocess import FramePreprocessor
+
+class TravLabel(FramePreprocessor):
+    output_key      = "trav_label"
+    output_loader   = "npys"
+    input_keys      = ["labels"]
+    timestamps_from = "labels"
+    sources         = ["labels"]
+
+    def process(self, sample) -> np.ndarray:
+        return (sample.data["labels"] < 10).astype(np.uint8)
+
+ds = apairo.Goose3DDataset("/data/goose", keys=["lidar", "labels"])
+ds.run_preprocess(TravLabel())
+```
+
+See [`apairo_preprocess`](https://github.com/apairo/apairo_preprocess) for a collection of ready-made preprocessors.
+
+---
+
+## Transforms
+
+Apply callables at access time — no disk writes.
+
+```python
+# Per-channel
+ds.transform("lidar", RangeFilter(max=50.0))
+
+# Sample-level — consistent mask across aligned channels
+def sync_filter(sample):
+    mask = np.linalg.norm(sample.data["lidar"][:, :3], axis=1) < 50.0
+    sample.data["lidar"]  = sample.data["lidar"][mask]
+    sample.data["labels"] = sample.data["labels"][mask]
+    return sample
+
+ds.transform(sync_filter)
+```
+
+See [`apairo_transform`](https://github.com/apairo/apairo_transform) for a collection of ready-made transforms.
+
+---
+
+## Filtering
+
+`filter()` returns a dataset view restricted to frames that pass a predicate. Sweep once, persist the indices, reload without I/O cost on subsequent runs:
+
+```python
+# Compute and save
+view = ds.filter("trav_gt", lambda gt: (gt == 1).sum() >= 50)
+np.save("cache/valid.npy", view.indices)
+
+# Reload — no sweep
+view = ds.filter(np.load("cache/valid.npy"))
+```
+
+---
+
+## Select & cache
+
+`select(keys)` narrows a dataset to a subset of channels. `cache()` materialises it in RAM. Together they let you cache only the channels worth caching:
+
+```python
+ds.transform("ground_height_csf", expensive_smooth)
+
+# Compute once, store in RAM
+ds_prior = ds.select(["ground_height_csf"]).cache()
+
+# Reuse across training runs — prior served from RAM, base channels from disk
+ds_v1 = base.join(ds_prior).transform(augment_v1)
+ds_v2 = base.join(ds_prior).transform(augment_v2)
+```
+
+---
+
+## Asynchronous datasets — `synchronize()`
+
+Asynchronous datasets (multi-rate sensor rigs) expose a timestamp-ordered event timeline: `ds[i]` is one event from one sensor. To get complete multi-channel frames, resample onto a reference clock:
+
+```python
+ds = apairo.TartanKittiDataset(seq_dir, keys=["velodyne_0", "image_left", "cmd"])
+
+ds_sync = ds.synchronize(
+    reference="velodyne_0",   # default: lowest-frequency channel
+    method="latest",          # "latest" (zero-order hold) or "nearest"
+    tolerance=0.05,           # drop frames with no match within ±50 ms
+)
+
+ds_sync[0].data   # {"velodyne_0": ..., "image_left": ..., "cmd": ...}
+```
+
+The result is a synchronous view — random access, shuffling, and the whole chaining API (`filter`, `select`, `cache`, `join`, `DataLoader`) work unchanged. Matching is a pure index computation; no data is read until access.
+
+---
+
+## Combining datasets
+
+```python
+# ConcatDataset — frame axis (different recording sessions)
+combined = apairo.ConcatDataset([ds_session1, ds_session2])
+
+# ZipDataset — channel axis (same frames, different modalities)
+combined = apairo.ZipDataset(ds_base, ds_prior)
+# or: ds_base.join(ds_prior)
+
+# Built-in splits
+ds_train = apairo.Rellis3DDataset(root, keys=["lidar", "labels"]).split("train")
+ds_val   = apairo.Rellis3DDataset(root, keys=["lidar", "labels"]).split("val")
+```
+
+---
+
+## Extending apairo
+
+Add a new synchronous dataset with a YAML profile and a minimal subclass.
+See [documentation](https://apairo-robotics.github.io/apairo/) for the full guide.
+
+---
+
+## Contributing
+
+apairo is one repository of a small ecosystem ([apairo_transform](https://github.com/apairo-robotics/apairo_transform), [apairo_preprocess](https://github.com/apairo-robotics/apairo_preprocess), [apairo_extractor](https://github.com/apairo-robotics/apairo_extractor), [apairo_rr](https://github.com/apairo-robotics/apairo_rr)). Where a change belongs, the design invariants, and the dev workflow are documented in [CONTRIBUTING.md](CONTRIBUTING.md).
+
+---
+
+## License
+
+MIT

apairo-0.2.0/README.md

@@ -0,0 +1,247 @@
+# apairo
+
+Unified Python loader for robotics sensor datasets — one API across synchronous and asynchronous layouts, with built-in preprocessing, filtering, and dataset composition.
+
+All data is returned as `numpy.ndarray`. Convert to the framework of your choice.
+
+---
+
+## Installation
+
+```bash
+pip install apairo
+```
+
+Optional extras:
+
+```bash
+pip install apairo[torch]    # PyTorch support (.pt files)
+pip install apairo[vision]   # Image loading (Pillow)
+```
+
+Requires Python ≥ 3.11.
+
+---
+
+## Quickstart
+
+```python
+import apairo
+
+ds = apairo.SemanticKittiDataset("/data/semantic_kitti", keys=["lidar", "labels"])
+sample = ds[0]
+# sample.data["lidar"]   -> np.ndarray (N, 4)  float32  [x, y, z, intensity]
+# sample.data["labels"]  -> np.ndarray (N,)    int64
+```
+
+---
+
+## Supported datasets
+
+| Class | Layout | Modalities |
+|---|---|---|
+| `SemanticKittiDataset` | synchronous | lidar, labels |
+| `Rellis3DDataset` | synchronous | lidar, labels, poses |
+| `Goose3DDataset` | synchronous | lidar, labels |
+| `MNTDataset` | synchronous | lidar, labels, poses |
+| `RawDataset` | asynchronous | any channels — declared in `.apairo/channels.yaml` |
+| `TartanKittiDataset` | asynchronous | any TartanDrive v2 channel |
+
+`RawDataset` is the profile-free loader for the asynchronous layout: it takes its
+channels — and their format (`npy`, `npys`, `bin`, `img`, `zarr`) — entirely from
+`.apairo/channels.yaml`, so it loads any such dataset, including the output of
+[apairo_extractor](https://github.com/apairo-robotics/apairo_extractor), with no
+code change.
+
+---
+
+## Command line
+
+Installing apairo provides the `apairo` command to inspect and initialize
+datasets from the terminal:
+
+```bash
+# Write/repair the .apairo sidecars by scanning a directory (root-aware, idempotent)
+apairo init /data/my_dataset
+
+# Show sequences, channels (tracked + untracked), event count and any issues
+apairo status /data/my_dataset           # add --json for machine output
+```
+
+`apairo init` reconstructs the `.apairo` files for data laid out before they
+existed (e.g. an older extraction) — no re-extraction needed — and the result
+loads directly with `RawDataset`. See [Command Line](https://apairo-robotics.github.io/apairo/cli/) for the full reference.
+
+---
+
+## Pipeline
+
+apairo provides a composable set of operations that chain together — each returns a full dataset:
+
+```python
+from apairo import Rellis3DDataset, FramePreprocessor
+from torch.utils.data import DataLoader
+import numpy as np
+
+# 1. Preprocess — run once, persisted in .apairo, reloaded transparently
+class TravLabel(FramePreprocessor):
+    output_key = "trav_gt";  output_loader = "npys"
+    input_keys = ["labels"]; timestamps_from = "lidar"; sources = ["labels"]
+    def process(self, sample): return (sample.data["labels"] < 10).astype(np.uint8)
+
+ds = Rellis3DDataset(root, keys=["lidar", "labels", "ground_height_csf"])
+ds.run_preprocess(TravLabel())
+
+# 2. Cache an expensive derived channel — computed once, served from RAM
+ds.transform("ground_height_csf", expensive_smooth)
+ds_prior = ds.select(["ground_height_csf"]).cache()
+
+# 3. Build train split — filter, join cached prior, apply augmentation
+valid = np.load("cache/valid_indices.npy")
+ds_train = (
+    Rellis3DDataset(root, keys=["lidar", "trav_gt"])
+    .filter(valid)
+    .join(ds_prior)
+    .transform("lidar", RangeFilter(max=50.0))
+)
+
+# 4. Drop into DataLoader — no adapter needed
+loader = DataLoader(ds_train, batch_size=8, shuffle=True, collate_fn=my_collate)
+```
+
+See [`examples/`](examples/) for complete runnable pipelines.
+
+---
+
+## Preprocessing
+
+Define a `FramePreprocessor` or `SequencePreprocessor`, run it once — apairo persists the output and reloads it transparently on subsequent runs.
+
+```python
+from apairo.preprocess import FramePreprocessor
+
+class TravLabel(FramePreprocessor):
+    output_key      = "trav_label"
+    output_loader   = "npys"
+    input_keys      = ["labels"]
+    timestamps_from = "labels"
+    sources         = ["labels"]
+
+    def process(self, sample) -> np.ndarray:
+        return (sample.data["labels"] < 10).astype(np.uint8)
+
+ds = apairo.Goose3DDataset("/data/goose", keys=["lidar", "labels"])
+ds.run_preprocess(TravLabel())
+```
+
+See [`apairo_preprocess`](https://github.com/apairo/apairo_preprocess) for a collection of ready-made preprocessors.
+
+---
+
+## Transforms
+
+Apply callables at access time — no disk writes.
+
+```python
+# Per-channel
+ds.transform("lidar", RangeFilter(max=50.0))
+
+# Sample-level — consistent mask across aligned channels
+def sync_filter(sample):
+    mask = np.linalg.norm(sample.data["lidar"][:, :3], axis=1) < 50.0
+    sample.data["lidar"]  = sample.data["lidar"][mask]
+    sample.data["labels"] = sample.data["labels"][mask]
+    return sample
+
+ds.transform(sync_filter)
+```
+
+See [`apairo_transform`](https://github.com/apairo/apairo_transform) for a collection of ready-made transforms.
+
+---
+
+## Filtering
+
+`filter()` returns a dataset view restricted to frames that pass a predicate. Sweep once, persist the indices, reload without I/O cost on subsequent runs:
+
+```python
+# Compute and save
+view = ds.filter("trav_gt", lambda gt: (gt == 1).sum() >= 50)
+np.save("cache/valid.npy", view.indices)
+
+# Reload — no sweep
+view = ds.filter(np.load("cache/valid.npy"))
+```
+
+---
+
+## Select & cache
+
+`select(keys)` narrows a dataset to a subset of channels. `cache()` materialises it in RAM. Together they let you cache only the channels worth caching:
+
+```python
+ds.transform("ground_height_csf", expensive_smooth)
+
+# Compute once, store in RAM
+ds_prior = ds.select(["ground_height_csf"]).cache()
+
+# Reuse across training runs — prior served from RAM, base channels from disk
+ds_v1 = base.join(ds_prior).transform(augment_v1)
+ds_v2 = base.join(ds_prior).transform(augment_v2)
+```
+
+---
+
+## Asynchronous datasets — `synchronize()`
+
+Asynchronous datasets (multi-rate sensor rigs) expose a timestamp-ordered event timeline: `ds[i]` is one event from one sensor. To get complete multi-channel frames, resample onto a reference clock:
+
+```python
+ds = apairo.TartanKittiDataset(seq_dir, keys=["velodyne_0", "image_left", "cmd"])
+
+ds_sync = ds.synchronize(
+    reference="velodyne_0",   # default: lowest-frequency channel
+    method="latest",          # "latest" (zero-order hold) or "nearest"
+    tolerance=0.05,           # drop frames with no match within ±50 ms
+)
+
+ds_sync[0].data   # {"velodyne_0": ..., "image_left": ..., "cmd": ...}
+```
+
+The result is a synchronous view — random access, shuffling, and the whole chaining API (`filter`, `select`, `cache`, `join`, `DataLoader`) work unchanged. Matching is a pure index computation; no data is read until access.
+
+---
+
+## Combining datasets
+
+```python
+# ConcatDataset — frame axis (different recording sessions)
+combined = apairo.ConcatDataset([ds_session1, ds_session2])
+
+# ZipDataset — channel axis (same frames, different modalities)
+combined = apairo.ZipDataset(ds_base, ds_prior)
+# or: ds_base.join(ds_prior)
+
+# Built-in splits
+ds_train = apairo.Rellis3DDataset(root, keys=["lidar", "labels"]).split("train")
+ds_val   = apairo.Rellis3DDataset(root, keys=["lidar", "labels"]).split("val")
+```
+
+---
+
+## Extending apairo
+
+Add a new synchronous dataset with a YAML profile and a minimal subclass.
+See [documentation](https://apairo-robotics.github.io/apairo/) for the full guide.
+
+---
+
+## Contributing
+
+apairo is one repository of a small ecosystem ([apairo_transform](https://github.com/apairo-robotics/apairo_transform), [apairo_preprocess](https://github.com/apairo-robotics/apairo_preprocess), [apairo_extractor](https://github.com/apairo-robotics/apairo_extractor), [apairo_rr](https://github.com/apairo-robotics/apairo_rr)). Where a change belongs, the design invariants, and the dev workflow are documented in [CONTRIBUTING.md](CONTRIBUTING.md).
+
+---
+
+## License
+
+MIT

apairo-0.2.0/apairo/__init__.py

@@ -0,0 +1,68 @@
+"""Apairo -- unified robotics dataset loader."""
+
+import logging
+
+from apairo.core.sample import Sample
+from apairo.core.synchronous_dataset import SynchronousDataset
+from apairo.core.configurable_dataset import ConfigurableDataset
+from apairo.preprocess import FramePreprocessor, SequencePreprocessor
+
+from apairo.dataset.raw import RawDataset
+from apairo.dataset.tartan_kitti import TartanKittiDataset
+from apairo.dataset.concat import ConcatDataset
+from apairo.dataset.zip import ZipDataset
+from apairo.dataset.stream import StreamDataset
+from apairo.dataset import split_sequences
+from apairo.core.sequence_view import SequenceView
+from apairo.core.filtered_view import FilteredView
+from apairo.core.channel_view import ChannelView
+from apairo.core.cached_dataset import CachedDataset
+from apairo.core.synchronized_view import SynchronizedView
+from apairo.core.interpolator import Interpolator
+from apairo.dataset.semantic_kitti import SemanticKittiDataset
+from apairo.dataset.rellis import Rellis3DDataset
+from apairo.dataset.goose import Goose3DDataset
+from apairo.dataset.mnt import MNTDataset
+
+from apairo.core.layout import ChannelSpec, DatasetLayout
+from apairo.core.transform import Compose
+from apairo.core.config import register_channel, register_raw_channel, verify_config
+from apairo.writer import WRITERS
+from apairo.loader import DERIVED_LOADERS
+
+logging.getLogger(__name__).addHandler(logging.NullHandler())
+
+__version__ = "0.2.0"
+
+__all__ = [
+    "Sample",
+    "SynchronousDataset",
+    "ConfigurableDataset",
+    "FramePreprocessor",
+    "SequencePreprocessor",
+    "RawDataset",
+    "TartanKittiDataset",
+    "ConcatDataset",
+    "ZipDataset",
+    "StreamDataset",
+    "split_sequences",
+    "SequenceView",
+    "FilteredView",
+    "ChannelView",
+    "CachedDataset",
+    "SynchronizedView",
+    "Interpolator",
+    "SemanticKittiDataset",
+    "Rellis3DDataset",
+    "Goose3DDataset",
+    "MNTDataset",
+    "ChannelSpec",
+    "DatasetLayout",
+    "Compose",
+    "register_channel",
+    "register_raw_channel",
+    "verify_config",
+    "WRITERS",
+    "DERIVED_LOADERS",
+    "__version__",
+]