parrot-codec 0.1.0__tar.gz

parrot_codec-0.1.0/Cargo.lock

@@ -0,0 +1,227 @@
+# This file is automatically @generated by Cargo.
+# It is not intended for manual editing.
+version = 4
+
+[[package]]
+name = "equivalent"
+version = "1.0.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f"
+
+[[package]]
+name = "hashbrown"
+version = "0.17.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a"
+
+[[package]]
+name = "heck"
+version = "0.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"
+
+[[package]]
+name = "indexmap"
+version = "2.14.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9"
+dependencies = [
+ "equivalent",
+ "hashbrown",
+]
+
+[[package]]
+name = "itoa"
+version = "1.0.18"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682"
+
+[[package]]
+name = "libc"
+version = "0.2.189"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2"
+
+[[package]]
+name = "memchr"
+version = "2.8.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98"
+
+[[package]]
+name = "once_cell"
+version = "1.21.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50"
+
+[[package]]
+name = "parrot_codec"
+version = "0.1.0"
+dependencies = [
+ "pyo3",
+ "serde_json",
+]
+
+[[package]]
+name = "portable-atomic"
+version = "1.14.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3d20d5497ef88037a52ff98267d066e7f11fcc5e99bbfbd58a42336193aacec3"
+
+[[package]]
+name = "proc-macro2"
+version = "1.0.107"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9"
+dependencies = [
+ "unicode-ident",
+]
+
+[[package]]
+name = "pyo3"
+version = "0.29.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cd274650b21d4bfc26a0a47587962c1edb425f69287324355cd040c3ea66071c"
+dependencies = [
+ "libc",
+ "once_cell",
+ "portable-atomic",
+ "pyo3-build-config",
+ "pyo3-ffi",
+ "pyo3-macros",
+]
+
+[[package]]
+name = "pyo3-build-config"
+version = "0.29.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c5e2a7d2f0d013342f295c048ad19237add5154a55b1c5a254c0ec93d4109078"
+dependencies = [
+ "target-lexicon",
+]
+
+[[package]]
+name = "pyo3-ffi"
+version = "0.29.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ca85c467da1bbc8d866eea5deff9cf29ea5f7785054a17da36e65bda9c05845b"
+dependencies = [
+ "libc",
+ "pyo3-build-config",
+]
+
+[[package]]
+name = "pyo3-macros"
+version = "0.29.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9ac53762fd065daa3194dd09337a38bd793a188100fd1a9304c4ab312d901771"
+dependencies = [
+ "proc-macro2",
+ "pyo3-macros-backend",
+ "quote",
+ "syn 2.0.119",
+]
+
+[[package]]
+name = "pyo3-macros-backend"
+version = "0.29.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4ca3a1557399783172dc5bf39cfca835157732532cba56b71d2292161e53b362"
+dependencies = [
+ "heck",
+ "proc-macro2",
+ "quote",
+ "syn 2.0.119",
+]
+
+[[package]]
+name = "quote"
+version = "1.0.47"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001"
+dependencies = [
+ "proc-macro2",
+]
+
+[[package]]
+name = "serde"
+version = "1.0.229"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba"
+dependencies = [
+ "serde_core",
+]
+
+[[package]]
+name = "serde_core"
+version = "1.0.229"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48"
+dependencies = [
+ "serde_derive",
+]
+
+[[package]]
+name = "serde_derive"
+version = "1.0.229"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 3.0.3",
+]
+
+[[package]]
+name = "serde_json"
+version = "1.0.151"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14"
+dependencies = [
+ "indexmap",
+ "itoa",
+ "memchr",
+ "serde",
+ "serde_core",
+ "zmij",
+]
+
+[[package]]
+name = "syn"
+version = "2.0.119"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "unicode-ident",
+]
+
+[[package]]
+name = "syn"
+version = "3.0.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "53e9bae58849f64dfa4f5d5ae372c8341f7305f82a3868709269343628b659a3"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "unicode-ident",
+]
+
+[[package]]
+name = "target-lexicon"
+version = "0.13.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "adb6935a6f5c20170eeceb1a3835a49e12e19d792f6dd344ccc76a985ca5a6ca"
+
+[[package]]
+name = "unicode-ident"
+version = "1.0.24"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75"
+
+[[package]]
+name = "zmij"
+version = "1.0.23"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b"

parrot_codec-0.1.0/Cargo.toml

@@ -0,0 +1,22 @@
+[package]
+name = "parrot_codec"
+version = "0.1.0"
+edition = "2021"
+description = "Optional Rust columnar-transform extension for parrot.tools.compression (FEAT-380)"
+
+[lib]
+name = "parrot_codec"
+crate-type = ["cdylib"]
+
+[dependencies]
+pyo3 = { version = "0.29", features = ["extension-module"] }
+# `preserve_order` keeps object-key insertion order matching the JSON text
+# order (the Python side's dict order) — required for the "first-seen
+# order across rows" determinism guarantee (G4); without it serde_json's
+# default Map is a sorted BTreeMap, which would silently reorder columns.
+serde_json = { version = "1.0", features = ["preserve_order"] }
+
+[profile.release]
+opt-level = 3
+lto = true
+codegen-units = 1

parrot_codec-0.1.0/PKG-INFO

@@ -0,0 +1,7 @@
+Metadata-Version: 2.4
+Name: parrot_codec
+Version: 0.1.0
+Classifier: Programming Language :: Rust
+Classifier: Programming Language :: Python :: Implementation :: CPython
+Summary: Optional Rust columnar-transform extension for parrot.tools.compression (FEAT-380)
+Requires-Python: >=3.8

parrot_codec-0.1.0/pyproject.toml

@@ -0,0 +1,18 @@
+[build-system]
+requires = ["maturin>=1.0,<2.0"]
+build-backend = "maturin"
+
+[project]
+name = "parrot_codec"
+version = "0.1.0"
+description = "Optional Rust columnar-transform extension for parrot.tools.compression (FEAT-380)"
+requires-python = ">=3.8"
+classifiers = [
+    "Programming Language :: Rust",
+    "Programming Language :: Python :: Implementation :: CPython",
+]
+
+[tool.maturin]
+module-name = "parrot_codec"
+bindings = "pyo3"
+features = ["pyo3/extension-module"]

parrot_codec-0.1.0/src/lib.rs

@@ -0,0 +1,300 @@
+//! File: codec-rs/src/lib.rs
+//!
+//! Optional Rust acceleration for the `columnar` compression codec
+//! (FEAT-380, TASK-1955). A single FFI crossing: JSON bytes/str in ->
+//! parse -> transform -> JSON bytes out, with the transform itself run
+//! under `py.allow_threads()` so the GIL is released for the duration.
+//!
+//! This module implements ONLY the core columnarization algorithm — the
+//! exact counterpart of `ColumnarCodec._columnarize()` in
+//! `parrot/tools/compression/codecs/columnar.py` (the Python
+//! implementation is the executable specification this must match
+//! byte-for-byte). All of the *outer* concerns (min_rows / heterogeneity /
+//! nested-value guards, QueryResult unwrapping, size measurement,
+//! `CompressionOutcome` construction) stay in Python — by the time this
+//! function is called, the Python side has already confirmed the input is
+//! eligible for columnarization.
+use std::panic::{catch_unwind, AssertUnwindSafe};
+
+use pyo3::exceptions::PyRuntimeError;
+use pyo3::prelude::*;
+use serde_json::{Map, Value};
+
+/// Python-`==`-compatible equality for `serde_json::Value`.
+///
+/// `serde_json::Value`'s derived `PartialEq` treats `Number(1)` (parsed
+/// from a JSON integer literal) and `Number(1.0)` (parsed from a JSON
+/// float literal) as UNEQUAL, whereas Python's `==` treats `1 == 1.0` as
+/// `True`. Left unfixed, a numerically-constant column mixing int/float
+/// literals across rows (e.g. `1` in row 0, `1.0` in row 1) would factor
+/// out as a constant in the Python reference implementation but NOT here
+/// — a byte-for-byte parity break (code-review fix, TASK-1955's parity
+/// contract). Compare as `f64` whenever both sides parse as numbers;
+/// fall back to `==` otherwise.
+fn values_equal_like_python(a: &Value, b: &Value) -> bool {
+    match (a, b) {
+        (Value::Number(x), Value::Number(y)) => match (x.as_f64(), y.as_f64()) {
+            (Some(fx), Some(fy)) => fx == fy,
+            _ => x == y,
+        },
+        _ => a == b,
+    }
+}
+
+/// Split row-oriented JSON (`[{...}, {...}, ...]`) into the columnar
+/// `{"columns": [...], "rows": [[...], ...], "constants": {...}}` form.
+///
+/// Column order is derived from the data (first-seen order across rows,
+/// in each row's own key order) — never sorted, matching the Python
+/// reference implementation's determinism guarantee (G4).
+///
+/// `min_rows` is accepted for parity with the Python codec's signature,
+/// but the min-rows guard is evaluated by the PYTHON caller before this
+/// function is ever invoked; it is not re-checked here (harmless no-op
+/// when the input already satisfies it).
+fn columnarize_json(payload: &[u8], _min_rows: usize) -> Result<Vec<u8>, String> {
+    let rows: Vec<Map<String, Value>> = serde_json::from_slice::<Vec<Value>>(payload)
+        .map_err(|e| format!("invalid row-oriented JSON: {e}"))?
+        .into_iter()
+        .map(|v| match v {
+            Value::Object(m) => Ok(m),
+            other => Err(format!("expected a JSON object row, got: {other}")),
+        })
+        .collect::<Result<Vec<_>, String>>()?;
+
+    // -- column extraction: first-seen order across rows ------------------
+    let mut columns: Vec<String> = Vec::new();
+    {
+        let mut seen = std::collections::HashSet::new();
+        for row in &rows {
+            for key in row.keys() {
+                if seen.insert(key.clone()) {
+                    columns.push(key.clone());
+                }
+            }
+        }
+    }
+
+    // -- null-column elision: a column that is null/absent in EVERY row --
+    let null_cols: Vec<String> = columns
+        .iter()
+        .filter(|c| {
+            rows.iter()
+                .all(|row| matches!(row.get(*c), None | Some(Value::Null)))
+        })
+        .cloned()
+        .collect();
+    columns.retain(|c| !null_cols.contains(c));
+
+    // -- constant-column factoring: only when there is more than one row --
+    let mut constants: Map<String, Value> = Map::new();
+    if rows.len() > 1 {
+        let mut factored: Vec<String> = Vec::new();
+        for col in columns.iter() {
+            let first = rows[0].get(col).cloned().unwrap_or(Value::Null);
+            let all_equal = rows.iter().all(|row| {
+                values_equal_like_python(&row.get(col).cloned().unwrap_or(Value::Null), &first)
+            });
+            if all_equal {
+                constants.insert(col.clone(), first);
+                factored.push(col.clone());
+            }
+        }
+        columns.retain(|c| !factored.contains(c));
+    }
+
+    // -- build positional rows, aligned to the final `columns` list -------
+    let out_rows: Vec<Vec<Value>> = rows
+        .iter()
+        .map(|row| {
+            columns
+                .iter()
+                .map(|c| row.get(c).cloned().unwrap_or(Value::Null))
+                .collect()
+        })
+        .collect();
+
+    let mut result = Map::new();
+    result.insert(
+        "columns".to_string(),
+        Value::Array(columns.into_iter().map(Value::String).collect()),
+    );
+    result.insert(
+        "rows".to_string(),
+        Value::Array(out_rows.into_iter().map(Value::Array).collect()),
+    );
+    result.insert("constants".to_string(), Value::Object(constants));
+
+    serde_json::to_vec(&Value::Object(result)).map_err(|e| format!("serialization error: {e}"))
+}
+
+/// Columnarize a row-oriented JSON payload (bytes or str), GIL released
+/// for the duration of the transform.
+///
+/// Args:
+///     payload: UTF-8 JSON bytes/str encoding a `list[dict]` of rows.
+///     min_rows: Accepted for parity with the Python codec's signature;
+///         not re-checked here (the Python caller already applied the
+///         guard before calling this function).
+///
+/// Returns:
+///     UTF-8 JSON bytes encoding `{"columns", "rows", "constants"}`.
+///
+/// Panic safety: `columnarize_json` is run inside `catch_unwind` so an
+/// unexpected panic (e.g. an unforeseen indexing bug) surfaces as an
+/// ordinary `PyRuntimeError` instead of PyO3's `PanicException`.
+/// `PanicException` derives from `BaseException`, not `Exception` — it
+/// would silently bypass `columnar.py`'s `except Exception:` guard and
+/// break the "a broken codec must never break a tool call" contract
+/// (code-review fix). This does NOT protect against a stack overflow,
+/// which aborts the process unconditionally in Rust and cannot be caught
+/// by any mechanism — a residual, documented limitation.
+#[pyfunction]
+#[pyo3(signature = (payload, min_rows=20))]
+fn columnarize(py: Python<'_>, payload: &[u8], min_rows: usize) -> PyResult<Vec<u8>> {
+    let owned = payload.to_vec();
+    // pyo3 0.29 renamed `Python::allow_threads` -> `Python::detach`
+    // (verified against this crate's pinned pyo3 version, `Cargo.toml`);
+    // functionally identical GIL-release mechanism (spec's own "Pattern
+    // to Follow" used the older `allow_threads` name).
+    let outcome = py.detach(move || {
+        catch_unwind(AssertUnwindSafe(|| columnarize_json(&owned, min_rows)))
+    });
+    let result = match outcome {
+        Ok(inner) => inner,
+        Err(panic_payload) => {
+            let msg = panic_payload
+                .downcast_ref::<&str>()
+                .map(|s| s.to_string())
+                .or_else(|| panic_payload.downcast_ref::<String>().cloned())
+                .unwrap_or_else(|| "unknown panic in columnarize_json".to_string());
+            Err(format!("internal panic during columnarization: {msg}"))
+        }
+    };
+    result.map_err(PyRuntimeError::new_err)
+}
+
+#[pymodule]
+fn parrot_codec(m: &Bound<'_, PyModule>) -> PyResult<()> {
+    m.add_function(wrap_pyfunction!(columnarize, m)?)?;
+    Ok(())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn run(input: &str) -> Value {
+        let out = columnarize_json(input.as_bytes(), 20).expect("transform failed");
+        serde_json::from_slice(&out).expect("invalid output JSON")
+    }
+
+    #[test]
+    fn column_order_is_first_seen_not_sorted() {
+        let out = run(r#"[{"z":1,"a":2},{"z":3,"a":4}]"#);
+        assert_eq!(out["columns"], serde_json::json!(["z", "a"]));
+    }
+
+    #[test]
+    fn null_columns_are_elided() {
+        let out = run(r#"[{"a":1,"b":null},{"a":2,"b":null}]"#);
+        assert_eq!(out["columns"], serde_json::json!(["a"]));
+    }
+
+    #[test]
+    fn missing_key_counts_as_null_for_elision() {
+        // "b" is entirely absent from every row -> same as always-null.
+        let out = run(r#"[{"a":1},{"a":2}]"#);
+        assert_eq!(out["columns"], serde_json::json!(["a"]));
+    }
+
+    #[test]
+    fn constant_columns_are_factored_when_more_than_one_row() {
+        let out = run(r#"[{"a":1,"region":"south"},{"a":2,"region":"south"}]"#);
+        assert_eq!(out["columns"], serde_json::json!(["a"]));
+        assert_eq!(out["constants"], serde_json::json!({"region": "south"}));
+    }
+
+    #[test]
+    fn single_row_never_factors_constants() {
+        let out = run(r#"[{"a":1,"b":2}]"#);
+        assert_eq!(out["constants"], serde_json::json!({}));
+        assert_eq!(out["columns"], serde_json::json!(["a", "b"]));
+    }
+
+    #[test]
+    fn rows_are_positionally_aligned_with_null_padding_for_missing_keys() {
+        let out = run(r#"[{"a":1,"c":3},{"a":2}]"#);
+        // "c" appears in row 0 only, and is not all-null (present+non-null
+        // in row 0) so it stays a column; row 1 is missing it -> null.
+        assert_eq!(out["columns"], serde_json::json!(["a", "c"]));
+        assert_eq!(out["rows"], serde_json::json!([[1, 3], [2, Value::Null]]));
+    }
+
+    #[test]
+    fn determinism_same_input_same_output() {
+        let input = r#"[{"a":1,"b":null,"region":"south"},{"a":2,"b":null,"region":"south"}]"#;
+        let first = run(input);
+        for _ in 0..25 {
+            assert_eq!(run(input), first);
+        }
+    }
+
+    #[test]
+    fn rejects_non_array_input() {
+        let err = columnarize_json(br#"{"not": "an array"}"#, 20);
+        assert!(err.is_err());
+    }
+
+    #[test]
+    fn rejects_non_object_rows() {
+        let err = columnarize_json(br#"[1, 2, 3]"#, 20);
+        assert!(err.is_err());
+    }
+
+    #[test]
+    fn values_equal_like_python_matches_python_semantics() {
+        assert!(values_equal_like_python(
+            &serde_json::json!(1),
+            &serde_json::json!(1.0)
+        ));
+        assert!(!values_equal_like_python(
+            &serde_json::json!(1),
+            &serde_json::json!(2)
+        ));
+        assert!(values_equal_like_python(
+            &serde_json::json!("a"),
+            &serde_json::json!("a")
+        ));
+        assert!(!values_equal_like_python(
+            &serde_json::json!("a"),
+            &serde_json::json!("b")
+        ));
+    }
+
+    #[test]
+    fn int_and_float_constants_are_treated_as_equal_like_python() {
+        // Mirrors Python's `1 == 1.0` (`True`) — a column holding `1` in
+        // one row and `1.0` in another must still factor out as a
+        // constant, matching `ColumnarCodec._columnarize()`'s Python `==`
+        // semantics (code-review fix: previously `Number(1) !=
+        // Number(1.0)` under serde_json's derived `PartialEq` broke this
+        // parity contract).
+        let out = run(r#"[{"a":1,"n":1},{"a":2,"n":1.0}]"#);
+        assert_eq!(out["columns"], serde_json::json!(["a"]));
+        assert_eq!(out["constants"]["n"], serde_json::json!(1));
+    }
+
+    #[test]
+    fn catch_unwind_around_transform_captures_panics() {
+        // Not a test of the `columnarize()` pyfunction itself (that needs
+        // a `Python` GIL token) — verifies the SAME `catch_unwind` +
+        // `AssertUnwindSafe` pattern used inside it correctly converts a
+        // panic into an `Err` instead of unwinding past the FFI boundary
+        // (code-review fix).
+        let outcome = catch_unwind(AssertUnwindSafe(|| -> Result<(), String> {
+            panic!("synthetic panic for test coverage");
+        }));
+        assert!(outcome.is_err());
+    }
+}