ferroload 0.1.0__tar.gz

ferroload-0.1.0/PKG-INFO

@@ -0,0 +1,98 @@
+Metadata-Version: 2.4
+Name: ferroload
+Version: 0.1.0
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Rust
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Classifier: Topic :: Software Development :: Libraries
+Requires-Dist: datasets>=2.14 ; extra == 'hf'
+Requires-Dist: pillow>=9 ; extra == 'hf'
+Requires-Dist: huggingface-hub>=0.20 ; extra == 'hf'
+Requires-Dist: torch>=2.0 ; extra == 'torch'
+Requires-Dist: numpy>=1.21 ; extra == 'torch'
+Provides-Extra: hf
+Provides-Extra: torch
+Summary: A pure-Rust multimodal dataset format + dataloader for PyTorch.
+Keywords: pytorch,dataloader,dataset,multimodal,rust,machine-learning,computer-vision,video,audio
+Author: Midhun
+License: Apache-2.0
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown; charset=UTF-8; variant=GFM
+Project-URL: Changelog, https://github.com/USERNAME/ferroload-rs/blob/main/CHANGELOG.md
+Project-URL: Documentation, https://USERNAME.github.io/ferroload-rs/
+Project-URL: Homepage, https://github.com/USERNAME/ferroload-rs
+Project-URL: Repository, https://github.com/USERNAME/ferroload-rs
+
+# ferroload
+
+**A pure-Rust multimodal dataset format + dataloader for PyTorch.**
+
+Ferroload stores images, video, audio, tensors, and rich metadata in a
+self-contained, shardable on-disk format and serves them to training loops with
+parallel decode in Rust (the GIL released), SQL-style subsetting, and a one-call
+loader that drops straight into PyTorch.
+
+## Install
+
+```bash
+pip install ferroload
+```
+
+Optional extras:
+
+```bash
+pip install "ferroload[hf]"      # HuggingFace import tooling (datasets, pillow, hub)
+pip install "ferroload[torch]"   # torch + numpy, for the DataLoader glue
+```
+
+In-Rust video decode is feature-gated (it needs system ffmpeg) and is **not** in
+the published wheel — build from source for it:
+
+```bash
+maturin develop --release --features video
+```
+
+## Quickstart
+
+```python
+from ferroload import make_loader
+
+dl = make_loader("/data/ds", batch_size=64,
+                 columns=["image", "video", "label"],   # kinds resolved from the manifest
+                 resize=(224, 224), out="torch")
+for epoch in range(epochs):
+    dl.set_epoch(epoch)                                  # reshuffle (DDP-aware)
+    for batch in dl:
+        train_step(batch)                                # batch["image"], batch["video"], batch["label"]
+```
+
+Enrich a dataset with an additive, resumable layer — functions bind to inputs
+positionally and run once per sample by default, so they're generic:
+
+```python
+import ferroload
+
+def mean_color(img):                                     # img <- inputs=["image"]
+    return img.mean(axis=(0, 1)).astype("float32")
+
+ds = ferroload.Dataset.open("/data/ds")
+ds = ds.map(mean_color, inputs=["image"],
+            outputs={"emb": ferroload.Modality("npy")}, name="emb")
+ds.read_array(0, "emb")
+```
+
+## Documentation
+
+Full docs (Python API, quickstart, Rust core, benchmarks) are built with MkDocs
+and published via GitHub Pages. See the project repository for links.
+
+## License
+
+Apache-2.0.
+

ferroload-0.1.0/README.md

@@ -0,0 +1,66 @@
+# ferroload
+
+**A pure-Rust multimodal dataset format + dataloader for PyTorch.**
+
+Ferroload stores images, video, audio, tensors, and rich metadata in a
+self-contained, shardable on-disk format and serves them to training loops with
+parallel decode in Rust (the GIL released), SQL-style subsetting, and a one-call
+loader that drops straight into PyTorch.
+
+## Install
+
+```bash
+pip install ferroload
+```
+
+Optional extras:
+
+```bash
+pip install "ferroload[hf]"      # HuggingFace import tooling (datasets, pillow, hub)
+pip install "ferroload[torch]"   # torch + numpy, for the DataLoader glue
+```
+
+In-Rust video decode is feature-gated (it needs system ffmpeg) and is **not** in
+the published wheel — build from source for it:
+
+```bash
+maturin develop --release --features video
+```
+
+## Quickstart
+
+```python
+from ferroload import make_loader
+
+dl = make_loader("/data/ds", batch_size=64,
+                 columns=["image", "video", "label"],   # kinds resolved from the manifest
+                 resize=(224, 224), out="torch")
+for epoch in range(epochs):
+    dl.set_epoch(epoch)                                  # reshuffle (DDP-aware)
+    for batch in dl:
+        train_step(batch)                                # batch["image"], batch["video"], batch["label"]
+```
+
+Enrich a dataset with an additive, resumable layer — functions bind to inputs
+positionally and run once per sample by default, so they're generic:
+
+```python
+import ferroload
+
+def mean_color(img):                                     # img <- inputs=["image"]
+    return img.mean(axis=(0, 1)).astype("float32")
+
+ds = ferroload.Dataset.open("/data/ds")
+ds = ds.map(mean_color, inputs=["image"],
+            outputs={"emb": ferroload.Modality("npy")}, name="emb")
+ds.read_array(0, "emb")
+```
+
+## Documentation
+
+Full docs (Python API, quickstart, Rust core, benchmarks) are built with MkDocs
+and published via GitHub Pages. See the project repository for links.
+
+## License
+
+Apache-2.0.

ferroload-0.1.0/ferroload-codec/Cargo.toml

@@ -0,0 +1,19 @@
+[package]
+name = "ferroload-codec"
+version = "0.1.0"
+edition = "2021"
+license = "Apache-2.0"
+description = "Modality decoders for Ferroload: image (pure-Rust), audio WAV (pure-Rust), video (ffmpeg/NVDEC, feature-gated)."
+
+[features]
+default = ["image-codec", "audio-codec"]
+image-codec = ["dep:image"]
+audio-codec = []                        # pure-Rust WAV PCM, no extra deps
+# Video decode links system ffmpeg (libav) / NVDEC and needs clang+pkg-config;
+# not buildable in every environment, so it is strictly opt-in.
+video-ffmpeg = ["dep:ffmpeg-next"]
+video-nvdec = ["video-ffmpeg"]
+
+[dependencies]
+image = { version = "0.25", optional = true, default-features = false, features = ["png", "jpeg"] }
+ffmpeg-next = { version = "7", optional = true }

ferroload-0.1.0/ferroload-codec/src/audio_wav.rs

@@ -0,0 +1,134 @@
+//! Pure-Rust WAV/PCM decoder -> `[channels, samples]` F32 in [-1, 1].
+//!
+//! Covers the uncompressed case (and precomputed-feature pipelines). Compressed
+//! formats (mp3/flac/aac) are intended to plug in via Symphonia behind an
+//! `audio-codecs` feature; the [`Codec`] surface is identical.
+
+use crate::{Codec, CodecError, Result, Tensor, TensorData};
+
+fn rd_u16(b: &[u8], o: usize) -> u16 {
+    u16::from_le_bytes([b[o], b[o + 1]])
+}
+fn rd_u32(b: &[u8], o: usize) -> u32 {
+    u32::from_le_bytes([b[o], b[o + 1], b[o + 2], b[o + 3]])
+}
+
+pub struct WavCodec;
+
+impl Codec for WavCodec {
+    fn decode(&self, b: &[u8]) -> Result<Tensor> {
+        if b.len() < 44 || &b[0..4] != b"RIFF" || &b[8..12] != b"WAVE" {
+            return Err(CodecError::Decode("not a RIFF/WAVE file".into()));
+        }
+        // walk chunks to find "fmt " and "data"
+        let mut pos = 12;
+        let (mut fmt, mut data): (Option<(u16, u16, u16)>, Option<(usize, usize)>) = (None, None);
+        while pos + 8 <= b.len() {
+            let id = &b[pos..pos + 4];
+            let sz = rd_u32(b, pos + 4) as usize;
+            let body = pos + 8;
+            if id == b"fmt " && body + 16 <= b.len() {
+                let audio_format = rd_u16(b, body);
+                let channels = rd_u16(b, body + 2);
+                let bits = rd_u16(b, body + 14);
+                fmt = Some((audio_format, channels, bits));
+            } else if id == b"data" {
+                let end = (body + sz).min(b.len());
+                data = Some((body, end));
+            }
+            pos = body + sz + (sz & 1); // chunks are word-aligned
+        }
+
+        let (audio_format, channels, bits) =
+            fmt.ok_or_else(|| CodecError::Decode("missing fmt chunk".into()))?;
+        let (ds, de) = data.ok_or_else(|| CodecError::Decode("missing data chunk".into()))?;
+        let channels = channels.max(1) as usize;
+        let raw = &b[ds..de];
+
+        // decode interleaved samples to f32
+        let interleaved: Vec<f32> = match (audio_format, bits) {
+            (1, 16) => raw
+                .chunks_exact(2)
+                .map(|c| i16::from_le_bytes([c[0], c[1]]) as f32 / 32768.0)
+                .collect(),
+            (1, 8) => raw.iter().map(|&x| (x as f32 - 128.0) / 128.0).collect(),
+            (3, 32) => raw
+                .chunks_exact(4)
+                .map(|c| f32::from_le_bytes([c[0], c[1], c[2], c[3]]))
+                .collect(),
+            (af, bps) => {
+                return Err(CodecError::Unsupported(format!(
+                    "WAV format={af} bits={bps} (use Symphonia for compressed audio)"
+                )))
+            }
+        };
+
+        let frames = interleaved.len() / channels;
+        // deinterleave -> channel-major [channels, frames]
+        let mut out = vec![0f32; channels * frames];
+        for f in 0..frames {
+            for c in 0..channels {
+                out[c * frames + f] = interleaved[f * channels + c];
+            }
+        }
+        Ok(Tensor {
+            shape: vec![channels, frames],
+            data: TensorData::F32(out),
+        })
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    /// Build a 16-bit PCM WAV from channel-major f32 samples.
+    fn make_wav(channels: u16, samples_per_ch: &[Vec<i16>]) -> Vec<u8> {
+        let frames = samples_per_ch[0].len();
+        let mut data = Vec::new();
+        for f in 0..frames {
+            for ch in 0..channels as usize {
+                data.extend_from_slice(&samples_per_ch[ch][f].to_le_bytes());
+            }
+        }
+        let mut w = Vec::new();
+        w.extend_from_slice(b"RIFF");
+        w.extend_from_slice(&(36 + data.len() as u32).to_le_bytes());
+        w.extend_from_slice(b"WAVE");
+        w.extend_from_slice(b"fmt ");
+        w.extend_from_slice(&16u32.to_le_bytes());
+        w.extend_from_slice(&1u16.to_le_bytes()); // PCM
+        w.extend_from_slice(&channels.to_le_bytes());
+        w.extend_from_slice(&16000u32.to_le_bytes()); // sample rate
+        w.extend_from_slice(&(16000 * channels as u32 * 2).to_le_bytes());
+        w.extend_from_slice(&(channels * 2).to_le_bytes());
+        w.extend_from_slice(&16u16.to_le_bytes()); // bits
+        w.extend_from_slice(b"data");
+        w.extend_from_slice(&(data.len() as u32).to_le_bytes());
+        w.extend_from_slice(&data);
+        w
+    }
+
+    #[test]
+    fn decode_stereo_pcm16() {
+        // ch0 = [0, 16384], ch1 = [-32768, 32767]
+        let wav = make_wav(2, &[vec![0, 16384], vec![-32768, 32767]]);
+        let t = WavCodec.decode(&wav).unwrap();
+        assert_eq!(t.shape, vec![2, 2]); // [channels, frames]
+        assert!(t.check());
+        if let TensorData::F32(d) = &t.data {
+            // channel-major: [ch0_f0, ch0_f1, ch1_f0, ch1_f1]
+            assert!((d[0] - 0.0).abs() < 1e-6);
+            assert!((d[1] - 0.5).abs() < 1e-6);
+            assert!((d[2] - (-1.0)).abs() < 1e-6);
+            assert!((d[3] - 0.9999).abs() < 1e-3);
+        } else {
+            panic!("expected F32");
+        }
+    }
+
+    #[test]
+    fn rejects_non_wav() {
+        assert!(WavCodec.decode(b"xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx").is_err());
+    }
+}

ferroload-0.1.0/ferroload-codec/src/image_codec.rs

@@ -0,0 +1,80 @@
+//! Pure-Rust image decoder (PNG/JPEG) producing an `[H, W, 3]` U8 tensor.
+
+use crate::{Codec, CodecError, Result, Tensor, TensorData};
+
+pub struct ImageCodec;
+
+impl ImageCodec {
+    /// Decode and resize to exactly `(h, w)` (RGB). Used for collation into a
+    /// uniform `[B,H,W,3]` batch (resize-on-decode avoids full-res then shrink).
+    pub fn decode_resized(&self, bytes: &[u8], h: usize, w: usize) -> Result<Tensor> {
+        let img = image::load_from_memory(bytes)
+            .map_err(|e| CodecError::Decode(format!("image: {e}")))?;
+        let img = img.resize_exact(w as u32, h as u32, image::imageops::FilterType::Triangle);
+        let data = img.to_rgb8().into_raw();
+        Ok(Tensor {
+            shape: vec![h, w, 3],
+            data: TensorData::U8(data),
+        })
+    }
+}
+
+impl Codec for ImageCodec {
+    fn decode(&self, bytes: &[u8]) -> Result<Tensor> {
+        let img = image::load_from_memory(bytes)
+            .map_err(|e| CodecError::Decode(format!("image: {e}")))?
+            .to_rgb8();
+        let (w, h) = img.dimensions();
+        let data = img.into_raw(); // row-major RGB, len = h*w*3
+        Ok(Tensor {
+            shape: vec![h as usize, w as usize, 3],
+            data: TensorData::U8(data),
+        })
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    // Encode a tiny RGB PNG in-memory, then decode it back.
+    fn make_png(w: u32, h: u32) -> Vec<u8> {
+        let mut buf = image::RgbImage::new(w, h);
+        for (x, y, px) in buf.enumerate_pixels_mut() {
+            *px = image::Rgb([x as u8, y as u8, 7]);
+        }
+        let mut out = std::io::Cursor::new(Vec::new());
+        image::DynamicImage::ImageRgb8(buf)
+            .write_to(&mut out, image::ImageFormat::Png)
+            .unwrap();
+        out.into_inner()
+    }
+
+    #[test]
+    fn decode_png_shape_and_pixels() {
+        let png = make_png(4, 3);
+        let t = ImageCodec.decode(&png).unwrap();
+        assert_eq!(t.shape, vec![3, 4, 3]); // [H, W, C]
+        assert!(t.check());
+        if let TensorData::U8(d) = &t.data {
+            // pixel (x=2,y=1) -> R=2,G=1,B=7 ; offset = (y*W + x)*3
+            let off = (1 * 4 + 2) * 3;
+            assert_eq!(&d[off..off + 3], &[2, 1, 7]);
+        } else {
+            panic!("expected U8");
+        }
+    }
+
+    #[test]
+    fn bad_bytes_error() {
+        assert!(ImageCodec.decode(b"not an image").is_err());
+    }
+
+    #[test]
+    fn decode_resized_to_fixed_shape() {
+        let png = make_png(10, 7);
+        let t = ImageCodec.decode_resized(&png, 4, 4).unwrap();
+        assert_eq!(t.shape, vec![4, 4, 3]); // [H, W, C] regardless of source size
+        assert!(t.check());
+    }
+}

ferroload-0.1.0/ferroload-codec/src/lib.rs

@@ -0,0 +1,111 @@
+//! # ferroload-codec
+//!
+//! Per-modality decoders behind a single [`Codec`] trait. A decoder turns raw
+//! member bytes (as stored in a shard) into a [`Tensor`]. Backends are
+//! feature-gated:
+//!
+//! - `image-codec` (default) — pure-Rust PNG/JPEG via the `image` crate.
+//! - `audio-codec` (default) — pure-Rust WAV/PCM decoder (no deps).
+//! - `video-ffmpeg` / `video-nvdec` (opt-in) — libav/NVDEC; require system
+//!   ffmpeg + clang, so they are not built in constrained environments.
+//!
+//! Unknown modalities with no registered codec can always fall back to raw bytes.
+
+use std::collections::BTreeMap;
+
+mod tensor;
+pub use tensor::{Dtype, Tensor, TensorData};
+
+#[cfg(feature = "image-codec")]
+pub mod image_codec;
+#[cfg(feature = "audio-codec")]
+pub mod audio_wav;
+pub mod sampling; // temporal frame-index selection (pure, always available)
+#[cfg(feature = "video-ffmpeg")]
+pub mod video;
+
+#[derive(Debug)]
+pub enum CodecError {
+    Decode(String),
+    Unsupported(String),
+}
+impl std::fmt::Display for CodecError {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self {
+            CodecError::Decode(s) => write!(f, "decode error: {s}"),
+            CodecError::Unsupported(s) => write!(f, "unsupported: {s}"),
+        }
+    }
+}
+impl std::error::Error for CodecError {}
+pub type Result<T> = std::result::Result<T, CodecError>;
+
+/// Decode raw member bytes into a tensor.
+pub trait Codec: Send + Sync {
+    fn decode(&self, bytes: &[u8]) -> Result<Tensor>;
+}
+
+/// Passthrough "codec" that returns the raw bytes as a 1-D U8 tensor — the
+/// fallback for modalities without a registered decoder.
+pub struct RawCodec;
+impl Codec for RawCodec {
+    fn decode(&self, bytes: &[u8]) -> Result<Tensor> {
+        Ok(Tensor {
+            shape: vec![bytes.len()],
+            data: TensorData::U8(bytes.to_vec()),
+        })
+    }
+}
+
+/// A registry mapping codec name -> implementation. Custom codecs (e.g. depth,
+/// hyperspectral) can be inserted at runtime.
+#[derive(Default)]
+pub struct Registry {
+    codecs: BTreeMap<String, Box<dyn Codec>>,
+}
+
+impl Registry {
+    pub fn new() -> Self {
+        Registry { codecs: BTreeMap::new() }
+    }
+
+    pub fn register(&mut self, name: &str, codec: Box<dyn Codec>) {
+        self.codecs.insert(name.to_string(), codec);
+    }
+
+    pub fn get(&self, name: &str) -> Option<&dyn Codec> {
+        self.codecs.get(name).map(|b| b.as_ref())
+    }
+
+    /// Decode with the named codec, falling back to raw bytes if unregistered.
+    pub fn decode_or_raw(&self, name: &str, bytes: &[u8]) -> Result<Tensor> {
+        match self.get(name) {
+            Some(c) => c.decode(bytes),
+            None => RawCodec.decode(bytes),
+        }
+    }
+
+    /// Registry preloaded with the compiled-in default codecs.
+    pub fn with_defaults() -> Self {
+        let mut r = Registry::new();
+        #[cfg(feature = "image-codec")]
+        r.register("image", Box::new(image_codec::ImageCodec));
+        #[cfg(feature = "audio-codec")]
+        r.register("audio", Box::new(audio_wav::WavCodec));
+        r.register("raw", Box::new(RawCodec));
+        r
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn raw_fallback_for_unknown_modality() {
+        let r = Registry::with_defaults();
+        let t = r.decode_or_raw("hyperspectral", b"\x01\x02\x03").unwrap();
+        assert_eq!(t.shape, vec![3]);
+        assert!(matches!(t.data, TensorData::U8(_)));
+    }
+}

ferroload-0.1.0/ferroload-codec/src/sampling.rs

@@ -0,0 +1,65 @@
+//! Temporal frame-index selection for video (DESIGN §14.5: subsample *which*
+//! frames to keep so we never decode-all-then-drop). Pure logic, fully tested;
+//! used by the feature-gated ffmpeg decoder to choose frames before decoding.
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum Sampling {
+    /// Evenly spaced across the clip.
+    Uniform,
+    /// Contiguous run starting at the front.
+    Dense,
+}
+
+/// Choose `num_frames` indices out of `total` according to `sampling`.
+/// Always returns indices in `[0, total)`, ascending, length `min(num_frames, total)`.
+pub fn frame_indices(total: usize, num_frames: usize, sampling: Sampling) -> Vec<usize> {
+    if total == 0 || num_frames == 0 {
+        return Vec::new();
+    }
+    let k = num_frames.min(total);
+    match sampling {
+        Sampling::Dense => (0..k).collect(),
+        Sampling::Uniform => {
+            if k == 1 {
+                return vec![total / 2];
+            }
+            // evenly spaced incl. endpoints: round(i*(total-1)/(k-1))
+            (0..k)
+                .map(|i| ((i * (total - 1)) as f64 / (k - 1) as f64).round() as usize)
+                .collect()
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn uniform_spans_endpoints() {
+        assert_eq!(frame_indices(10, 4, Sampling::Uniform), vec![0, 3, 6, 9]);
+        assert_eq!(frame_indices(100, 1, Sampling::Uniform), vec![50]);
+    }
+
+    #[test]
+    fn dense_is_prefix() {
+        assert_eq!(frame_indices(10, 4, Sampling::Dense), vec![0, 1, 2, 3]);
+    }
+
+    #[test]
+    fn clamps_when_fewer_frames_than_requested() {
+        assert_eq!(frame_indices(3, 8, Sampling::Uniform), vec![0, 1, 2]);
+        assert_eq!(frame_indices(0, 8, Sampling::Uniform), Vec::<usize>::new());
+    }
+
+    #[test]
+    fn indices_in_range_and_sorted() {
+        for total in [1usize, 5, 16, 257] {
+            for nf in [1usize, 4, 16] {
+                let idx = frame_indices(total, nf, Sampling::Uniform);
+                assert!(idx.iter().all(|&i| i < total));
+                assert!(idx.windows(2).all(|w| w[0] <= w[1]));
+            }
+        }
+    }
+}

ferroload-0.1.0/ferroload-codec/src/tensor.rs

@@ -0,0 +1,42 @@
+//! Minimal owned tensor returned by decoders. The PyO3 layer maps these to
+//! torch tensors (zero-copy via DLPack) at the boundary.
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum Dtype {
+    U8,
+    F32,
+}
+
+#[derive(Debug, Clone, PartialEq)]
+pub enum TensorData {
+    U8(Vec<u8>),
+    F32(Vec<f32>),
+}
+
+#[derive(Debug, Clone, PartialEq)]
+pub struct Tensor {
+    pub shape: Vec<usize>,
+    pub data: TensorData,
+}
+
+impl Tensor {
+    pub fn dtype(&self) -> Dtype {
+        match self.data {
+            TensorData::U8(_) => Dtype::U8,
+            TensorData::F32(_) => Dtype::F32,
+        }
+    }
+
+    pub fn numel(&self) -> usize {
+        self.shape.iter().product()
+    }
+
+    /// Validate that the element count matches the declared shape.
+    pub fn check(&self) -> bool {
+        let n = self.numel();
+        match &self.data {
+            TensorData::U8(v) => v.len() == n,
+            TensorData::F32(v) => v.len() == n,
+        }
+    }
+}