rypipe 2.1.0__tar.gz → 2.2.0__tar.gz

{rypipe-2.1.0 → rypipe-2.2.0}/Cargo.lock

@@ -417,9 +417,9 @@ checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04"
 
 [[package]]
 name = "cc"
-version = "1.4.4"
+version = "1.4.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0ad534f4357a5264cce5019c989cf66a4f0dc4e0d1b1d15f8aacec0ff7360273"
+checksum = "005ec2760ca554fae18df7a11195552ec576cd665632a881bc011d5bb2fd4d80"
 dependencies = [
  "find-msvc-tools",
  "jobserver",
@@ -518,9 +518,9 @@ checksum = "252afb9ae5eaa683babdc6a068b3f5726eb19e05070c731f9b2a23a7c3e8ed34"
 
 [[package]]
 name = "find-msvc-tools"
-version = "0.1.11"
+version = "0.1.12"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d45db016d36b838f563236e9193d0ee6ce38f3f68b6c94e914b4929c96bbb890"
+checksum = "3e0f1c7c3a72c66fd80abe965175f7523475c0489a87d3ff9d6e8c87d87a9d2d"
 
 [[package]]
 name = "flate2"
@@ -1058,7 +1058,7 @@ checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f"
 
 [[package]]
 name = "rypipe-core"
-version = "2.1.0"
+version = "2.2.0"
 dependencies = [
  "arrow 55.2.0",
  "chrono",
@@ -1078,7 +1078,7 @@ dependencies = [
 
 [[package]]
 name = "rypipe-python"
-version = "2.1.0"
+version = "2.2.0"
 dependencies = [
  "arrow 59.3.0",
  "mimalloc",

{rypipe-2.1.0 → rypipe-2.2.0}/Cargo.toml

@@ -3,7 +3,7 @@ members = ["crates/rypipe-core", "crates/rypipe-python"]
 resolver = "2"
 
 [workspace.package]
-version = "2.1.0"
+version = "2.2.0"
 edition = "2021"
 license = "MIT"
 repository = "https://github.com/emiliano-go/rypipe"

{rypipe-2.1.0 → rypipe-2.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rypipe
-Version: 2.1.0
+Version: 2.2.0
 Requires-Dist: pyarrow>=15
 Requires-Dist: rypipe[pandas,polars] ; extra == 'all'
 Requires-Dist: pytest>=7 ; extra == 'dev'

{rypipe-2.1.0 → rypipe-2.2.0}/crates/rypipe-core/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "rypipe-core"
-version = "2.1.0"
+version = "2.2.0"
 edition = "2021"
 license = "MIT"
 description = "Format-agnostic ingestion framework with Arrow export"

{rypipe-2.1.0 → rypipe-2.2.0}/crates/rypipe-core/src/columnar.rs

@@ -454,20 +454,31 @@ pub(crate) enum ColumnBuilder {
     Timestamp(TimeUnit, PrimColumn<i64>),
     Dictionary {
         codes: NullableColumn<i32>,
-        dict: Vec<String>,
+        /// Contiguous byte buffer for dictionary values (like StrColumn).
+        data: Vec<u8>,
+        /// Byte offsets into `data` for each dictionary entry.
+        offsets: Vec<i32>,
         /// value → code side-index.
-        index: HashMap<String, i32>,
+        index: rustc_hash::FxHashMap<Box<str>, i32>,
     },
 }
 
 /// Look up `v` in the dictionary index, inserting a new code if absent.
-fn dict_code(dict: &mut Vec<String>, index: &mut HashMap<String, i32>, v: &str) -> i32 {
-    if let Some(&code) = index.get(v) {
+/// Uses a contiguous byte buffer for values instead of Vec<String>.
+fn dict_code(
+    data: &mut Vec<u8>,
+    offsets: &mut Vec<i32>,
+    index: &mut rustc_hash::FxHashMap<Box<str>, i32>,
+    v: &str,
+) -> i32 {
+    let key: Box<str> = v.into();
+    if let Some(&code) = index.get(&key) {
         return code;
     }
-    let code = dict.len() as i32;
-    dict.push(v.to_owned());
-    index.insert(v.to_owned(), code);
+    let code = offsets.len() as i32 - 1;
+    data.extend_from_slice(v.as_bytes());
+    offsets.push(data.len() as i32);
+    index.insert(key, code);
     code
 }
 
@@ -547,8 +558,9 @@ impl ColumnBuilder {
             }
             FieldType::Dictionary => ColumnBuilder::Dictionary {
                 codes: NullableColumn::with_capacity(cap),
-                dict: Vec::new(),
-                index: HashMap::default(),
+                data: Vec::new(),
+                offsets: vec![0],
+                index: rustc_hash::FxHashMap::default(),
             },
         }
     }
@@ -567,8 +579,8 @@ impl ColumnBuilder {
                 ColumnBuilder::Int64(v) => v.push(Some(i)),
                 ColumnBuilder::Float64(v) => v.push(Some(i as f64)),
                 ColumnBuilder::String(col) => col.push(Some(&i.to_string())),
-                ColumnBuilder::Dictionary { codes, dict, index } => {
-                    let code = dict_code(dict, index, &i.to_string());
+                ColumnBuilder::Dictionary { codes, data, offsets, index } => {
+                    let code = dict_code(data, offsets, index, &i.to_string());
                     codes.push(Some(code));
                 }
                 _ => self.push(None),
@@ -577,8 +589,8 @@ impl ColumnBuilder {
                 ColumnBuilder::Float64(v) => v.push(Some(f)),
                 ColumnBuilder::Int64(v) => v.push(Some(f as i64)),
                 ColumnBuilder::String(col) => col.push(Some(&f.to_string())),
-                ColumnBuilder::Dictionary { codes, dict, index } => {
-                    let code = dict_code(dict, index, &f.to_string());
+                ColumnBuilder::Dictionary { codes, data, offsets, index } => {
+                    let code = dict_code(data, offsets, index, &f.to_string());
                     codes.push(Some(code));
                 }
                 _ => self.push(None),
@@ -586,8 +598,8 @@ impl ColumnBuilder {
             Value::Bool(b) => match self {
                 ColumnBuilder::Boolean(v) => v.push(Some(b)),
                 ColumnBuilder::String(col) => col.push(Some(&b.to_string())),
-                ColumnBuilder::Dictionary { codes, dict, index } => {
-                    let code = dict_code(dict, index, &b.to_string());
+                ColumnBuilder::Dictionary { codes, data, offsets, index } => {
+                    let code = dict_code(data, offsets, index, &b.to_string());
                     codes.push(Some(code));
                 }
                 _ => self.push(None),
@@ -597,8 +609,8 @@ impl ColumnBuilder {
                 ColumnBuilder::Int64(v) => v.push(Some(d as i64)),
                 ColumnBuilder::Float64(v) => v.push(Some(d as f64)),
                 ColumnBuilder::String(col) => col.push(Some(&format_date32(d))),
-                ColumnBuilder::Dictionary { codes, dict, index } => {
-                    let code = dict_code(dict, index, &format_date32(d));
+                ColumnBuilder::Dictionary { codes, data, offsets, index } => {
+                    let code = dict_code(data, offsets, index, &format_date32(d));
                     codes.push(Some(code));
                 }
                 _ => self.push(None),
@@ -614,12 +626,12 @@ impl ColumnBuilder {
                         Some(unit) => col.push(Some(&format_timestamp(ts, unit))),
                         None => col.push(Some(&ts.to_string())),
                     },
-                    ColumnBuilder::Dictionary { codes, dict, index } => {
+                    ColumnBuilder::Dictionary { codes, data, offsets, index } => {
                         let text = match unit {
                             Some(unit) => format_timestamp(ts, unit),
                             None => ts.to_string(),
                         };
-                        let code = dict_code(dict, index, &text);
+                        let code = dict_code(data, offsets, index, &text);
                         codes.push(Some(code));
                     }
                     _ => self.push(None),
@@ -697,16 +709,17 @@ impl ColumnBuilder {
                     "cannot promote column to unified variant '{target}'"
                 )));
             }
-            let mut dict: Vec<String> = Vec::new();
-            let mut index: HashMap<String, i32> = HashMap::default();
+            let mut data: Vec<u8> = Vec::new();
+            let mut offsets: Vec<i32> = vec![0];
+            let mut index: rustc_hash::FxHashMap<Box<str>, i32> = rustc_hash::FxHashMap::default();
             let mut codes = NullableColumn::with_capacity(old.len());
             for val in old.iter() {
                 match val {
-                    Some(s) => codes.push(Some(dict_code(&mut dict, &mut index, s))),
+                    Some(s) => codes.push(Some(dict_code(&mut data, &mut offsets, &mut index, s))),
                     None => codes.push(None),
                 }
             }
-            *self = ColumnBuilder::Dictionary { codes, dict, index };
+            *self = ColumnBuilder::Dictionary { codes, data, offsets, index };
             return Ok(());
         }
 
@@ -735,9 +748,9 @@ impl ColumnBuilder {
             ColumnBuilder::Timestamp(unit, v) => {
                 v.push(value.and_then(|s| parse_timestamp(&s, *unit)));
             }
-            ColumnBuilder::Dictionary { codes, dict, index } => match value {
+            ColumnBuilder::Dictionary { codes, data, offsets, index } => match value {
                 Some(v) => {
-                    let idx = dict_code(dict, index, &v);
+                    let idx = dict_code(data, offsets, index, &v);
                     codes.push(Some(idx));
                 }
                 None => codes.push(None),
@@ -765,9 +778,9 @@ impl ColumnBuilder {
             ColumnBuilder::Timestamp(unit, v) => {
                 v.push(value.and_then(|s| parse_timestamp(s, *unit)));
             }
-            ColumnBuilder::Dictionary { codes, dict, index } => match value {
+            ColumnBuilder::Dictionary { codes, data, offsets, index } => match value {
                 Some(v) => {
-                    let idx = dict_code(dict, index, v);
+                    let idx = dict_code(data, offsets, index, v);
                     codes.push(Some(idx));
                 }
                 None => codes.push(None),
@@ -807,7 +820,7 @@ impl ColumnBuilder {
             ColumnBuilder::Boolean(v) => v.bytes_used(),
             ColumnBuilder::Date32(v) => v.bytes_used(),
             ColumnBuilder::Timestamp(_, v) => v.bytes_used(),
-            ColumnBuilder::Dictionary { codes, dict, .. } => codes.len() * 4 + dict.len() * 16,
+            ColumnBuilder::Dictionary { codes, data, offsets, .. } => codes.len() * 4 + data.len() + offsets.len() * 4,
         }
     }
 
@@ -820,8 +833,8 @@ impl ColumnBuilder {
             ColumnBuilder::Boolean(v) => v.capacity_bytes(),
             ColumnBuilder::Date32(v) => v.capacity_bytes(),
             ColumnBuilder::Timestamp(_, v) => v.capacity_bytes(),
-            ColumnBuilder::Dictionary { codes, dict, .. } => {
-                codes.capacity_bytes() + dict.capacity() * 16
+            ColumnBuilder::Dictionary { codes, data, offsets, .. } => {
+                codes.capacity_bytes() + data.capacity() + offsets.capacity() * 4
             }
         }
     }
@@ -835,11 +848,12 @@ impl ColumnBuilder {
             ColumnBuilder::Boolean(v) => ColumnBuilder::Boolean(v.split_off(n)),
             ColumnBuilder::Date32(v) => ColumnBuilder::Date32(v.split_off(n)),
             ColumnBuilder::Timestamp(unit, v) => ColumnBuilder::Timestamp(*unit, v.split_off(n)),
-            ColumnBuilder::Dictionary { codes, dict, index } => {
+            ColumnBuilder::Dictionary { codes, data, offsets, index } => {
                 let other_codes = codes.split_off(n);
                 ColumnBuilder::Dictionary {
                     codes: other_codes,
-                    dict: dict.clone(),
+                    data: data.clone(),
+                    offsets: offsets.clone(),
                     index: index.clone(),
                 }
             }
@@ -852,8 +866,12 @@ impl ColumnBuilder {
     pub(crate) fn get_filter_view(&self, index: usize) -> Option<&str> {
         match self {
             ColumnBuilder::String(v) => v.get(index),
-            ColumnBuilder::Dictionary { codes, dict, .. } => {
-                codes.get(index).map(|code| dict[*code as usize].as_str())
+            ColumnBuilder::Dictionary { codes, data, offsets, .. } => {
+                codes.get(index).map(|code| {
+                    let start = offsets[*code as usize] as usize;
+                    let end = offsets[*code as usize + 1] as usize;
+                    std::str::from_utf8(&data[start..end]).unwrap_or("")
+                })
             }
             _ => None,
         }
@@ -872,8 +890,12 @@ impl ColumnBuilder {
                 let unit = *unit;
                 v.get(index).map(|ts| format_timestamp(ts, unit))
             }
-            ColumnBuilder::Dictionary { codes, dict, .. } => {
-                codes.get(index).map(|code| dict[*code as usize].clone())
+            ColumnBuilder::Dictionary { codes, data, offsets, .. } => {
+                codes.get(index).map(|code| {
+                    let start = offsets[*code as usize] as usize;
+                    let end = offsets[*code as usize + 1] as usize;
+                    String::from_utf8_lossy(&data[start..end]).into_owned()
+                })
             }
         }
     }
@@ -888,9 +910,13 @@ impl ColumnBuilder {
             ColumnBuilder::Boolean(v) => v.get(index).map(TypedValue::Bool),
             ColumnBuilder::Date32(v) => v.get(index).map(TypedValue::Date32),
             ColumnBuilder::Timestamp(_, v) => v.get(index).map(TypedValue::Timestamp),
-            ColumnBuilder::Dictionary { codes, dict, .. } => codes
+            ColumnBuilder::Dictionary { codes, data, offsets, .. } => codes
                 .get(index)
-                .map(|&idx| TypedValue::Str(dict[idx as usize].as_str())),
+                .map(|&idx| {
+                    let start = offsets[idx as usize] as usize;
+                    let end = offsets[idx as usize + 1] as usize;
+                    TypedValue::Str(std::str::from_utf8(&data[start..end]).unwrap_or(""))
+                }),
         }
     }
 
@@ -925,20 +951,25 @@ impl ColumnBuilder {
             (
                 ColumnBuilder::Dictionary {
                     codes: a_codes,
-                    dict: a_dict,
+                    data: a_data,
+                    offsets: a_offsets,
                     index: a_index,
                 },
                 ColumnBuilder::Dictionary {
                     codes: b_codes,
-                    dict: b_dict,
+                    data: b_data,
+                    offsets: b_offsets,
                     ..
                 },
             ) => {
                 // Remap b's dictionary into a's once, then translate codes.
-                let remap: Vec<i32> = b_dict
-                    .iter()
-                    .map(|val| dict_code(a_dict, a_index, val))
-                    .collect();
+                let mut remap: Vec<i32> = Vec::with_capacity(b_offsets.len() - 1);
+                for i in 0..(b_offsets.len() - 1) {
+                    let start = b_offsets[i] as usize;
+                    let end = b_offsets[i + 1] as usize;
+                    let val = std::str::from_utf8(&b_data[start..end]).unwrap_or("");
+                    remap.push(dict_code(a_data, a_offsets, a_index, val));
+                }
                 for c in b_codes.iter() {
                     a_codes.push(c.map(|code| remap[*code as usize]));
                 }
@@ -981,19 +1012,20 @@ impl ColumnBuilder {
             return;
         }
         // Upgrade: build dictionary + codes.
-        let mut dict: Vec<String> = Vec::new();
-        let mut index: HashMap<String, i32> = HashMap::default();
+        let mut data: Vec<u8> = Vec::new();
+        let mut offsets: Vec<i32> = vec![0];
+        let mut index: rustc_hash::FxHashMap<Box<str>, i32> = rustc_hash::FxHashMap::default();
         let mut codes = NullableColumn::with_capacity(old.len());
         for v in old.iter() {
             match v {
                 Some(s) => {
-                    let idx = dict_code(&mut dict, &mut index, s);
+                    let idx = dict_code(&mut data, &mut offsets, &mut index, s);
                     codes.push(Some(idx));
                 }
                 None => codes.push(None),
             }
         }
-        *self = ColumnBuilder::Dictionary { codes, dict, index };
+        *self = ColumnBuilder::Dictionary { codes, data, offsets, index };
     }
 
     /// Mutable access to dictionary codes for in-place remap.
@@ -1009,11 +1041,13 @@ impl ColumnBuilder {
     /// No-op if not a Dictionary variant.
     pub(crate) fn replace_dict(
         &mut self,
-        new_dict: Vec<String>,
-        new_index: rustc_hash::FxHashMap<String, i32>,
+        new_data: Vec<u8>,
+        new_offsets: Vec<i32>,
+        new_index: rustc_hash::FxHashMap<Box<str>, i32>,
     ) {
-        if let ColumnBuilder::Dictionary { dict, index, .. } = self {
-            *dict = new_dict;
+        if let ColumnBuilder::Dictionary { data, offsets, index, .. } = self {
+            *data = new_data;
+            *offsets = new_offsets;
             *index = new_index;
         }
     }
@@ -1100,13 +1134,14 @@ impl ColumnBuilder {
                 TimeUnit::Microsecond => v.to_arrow::<TimestampMicrosecondType>()?,
                 TimeUnit::Nanosecond => v.to_arrow::<TimestampNanosecondType>()?,
             },
-            ColumnBuilder::Dictionary { codes, dict, .. } => {
+            ColumnBuilder::Dictionary { codes, data, offsets, .. } => {
+                use arrow::buffer::{Buffer, OffsetBuffer, ScalarBuffer};
                 let codes_arr: Int32Array = std::mem::take(codes).into_options().collect();
-                let values: ArrayRef = Arc::new(
-                    dict.iter()
-                        .map(|s| Some(s.as_str()))
-                        .collect::<StringArray>(),
-                );
+                // Zero-copy: wrap contiguous data+offsets directly into StringArray.
+                // Dictionary values are always non-null, so validity is None.
+                let offsets_buf = OffsetBuffer::new(ScalarBuffer::from(std::mem::take(offsets)));
+                let data_buf = Buffer::from_vec(std::mem::take(data));
+                let values: ArrayRef = Arc::new(StringArray::try_new(offsets_buf, data_buf, None)?);
                 let arr = DictionaryArray::<Int32Type>::try_new(codes_arr, values)?;
                 Arc::new(arr)
             }
@@ -1277,8 +1312,16 @@ mod tests {
         d.push_value(Value::Float64(7.0)); // same string as Int64(7)
         d.push_value(Value::Bool(true));
         d.push_value(Value::Null);
-        if let ColumnBuilder::Dictionary { codes, dict, .. } = &d {
-            assert_eq!(dict, &["7", "true"]);
+        if let ColumnBuilder::Dictionary { codes, data, offsets, .. } = &d {
+            // Verify dictionary values via the contiguous buffer
+            let vals: Vec<&str> = (0..offsets.len() - 1)
+                .map(|i| {
+                    let start = offsets[i] as usize;
+                    let end = offsets[i + 1] as usize;
+                    std::str::from_utf8(&data[start..end]).unwrap()
+                })
+                .collect();
+            assert_eq!(vals, vec!["7", "true"]);
             assert_eq!(
                 codes.iter().map(|o| o.copied()).collect::<Vec<_>>(),
                 vec![Some(0), Some(0), Some(1), None]

{rypipe-2.1.0 → rypipe-2.2.0}/crates/rypipe-core/src/dict.rs

@@ -74,7 +74,7 @@ pub struct DictValues {
 
 /// Unify per-chunk dictionaries with the seed.
 ///
-/// Works with `ColumnBuilder::Dictionary { codes, dict, index }`.
+/// Works with `ColumnBuilder::Dictionary { codes, data, offsets, index }`.
 /// Seed codes are `0..seed.len()`, overflow are `seed.len()..`.
 pub(crate) fn unify_dictionaries(
     seed: &SeedDict,
@@ -91,12 +91,15 @@ pub(crate) fn unify_dictionaries(
 
     // Collect overflow values not in seed
     for col in locals {
-        if let ColumnBuilder::Dictionary { dict, .. } = col {
-            for s in dict.iter().skip(seed.len()) {
-                let k: Box<str> = s.clone().into_boxed_str();
+        if let ColumnBuilder::Dictionary { data, offsets, .. } = col {
+            for i in seed.len()..(offsets.len() - 1) {
+                let start = offsets[i] as usize;
+                let end = offsets[i + 1] as usize;
+                let val = std::str::from_utf8(&data[start..end]).unwrap_or("");
+                let k: Box<str> = val.into();
                 if !global_index.contains_key(&k) {
                     global_index.insert(k.clone(), next_code);
-                    global_data.extend_from_slice(s.as_bytes());
+                    global_data.extend_from_slice(val.as_bytes());
                     global_offsets.push(global_data.len() as i32);
                     next_code += 1;
                 }
@@ -112,12 +115,15 @@ pub(crate) fn unify_dictionaries(
     // Build remap tables: local code -> global code
     let mut remaps = Vec::with_capacity(locals.len());
     for col in locals {
-        if let ColumnBuilder::Dictionary { dict, index, .. } = col {
-            let local_len = dict.len();
+        if let ColumnBuilder::Dictionary { data, offsets, index, .. } = col {
+            let local_len = offsets.len() - 1;
             let mut map = Vec::with_capacity(local_len);
             let mut is_identity = true;
-            for s in dict {
-                let k: Box<str> = s.clone().into_boxed_str();
+            for i in 0..local_len {
+                let start = offsets[i] as usize;
+                let end = offsets[i + 1] as usize;
+                let val = std::str::from_utf8(&data[start..end]).unwrap_or("");
+                let k: Box<str> = val.into();
                 let g = *global_index.get(&k).unwrap();
                 let local = *index.get(k.as_ref()).unwrap();
                 if g != local {
@@ -126,9 +132,9 @@ pub(crate) fn unify_dictionaries(
                 map.push(g);
             }
             // If no overflow, it's essentially identity if dict == seed
-            if dict.len() == seed.len() && is_identity {
+            if local_len == seed.len() && is_identity {
                 // All seed, identity
-            } else if dict.len() > seed.len() {
+            } else if local_len > seed.len() {
                 is_identity = false;
             }
             remaps.push(RemapTable { map, is_identity });

{rypipe-2.1.0 → rypipe-2.2.0}/crates/rypipe-core/src/parallel.rs

@@ -153,21 +153,21 @@ impl ParallelExecutor {
                     let first_idx = engines[0].field_index.get(col_name).copied();
                     if let Some(idx) = first_idx {
                         if engines[0].columns[idx].variant_key() == "dictionary" {
-                            let first_dict =
-                                if let crate::columnar::ColumnBuilder::Dictionary { dict, .. } =
+                            let first_data =
+                                if let crate::columnar::ColumnBuilder::Dictionary { data, offsets, .. } =
                                     &engines[0].columns[idx]
                                 {
-                                    dict
+                                    (data, offsets)
                                 } else {
                                     continue;
                                 };
                             for e in &engines[1..] {
                                 if let Some(&j) = e.field_index.get(col_name) {
                                     if let crate::columnar::ColumnBuilder::Dictionary {
-                                        dict, ..
+                                        data, offsets, ..
                                     } = &e.columns[j]
                                     {
-                                        if dict != first_dict {
+                                        if data != first_data.0 || offsets != first_data.1 {
                                             unify_col = Some(col_name.clone());
                                             break;
                                         }
@@ -185,42 +185,54 @@ impl ParallelExecutor {
                 }
                 let col_name = unify_col.unwrap();
                 // Build seed from first chunk, unify, remap.
-                let first_dict =
-                    if let crate::columnar::ColumnBuilder::Dictionary { dict, .. } =
+                let first_data =
+                    if let crate::columnar::ColumnBuilder::Dictionary { data, offsets, .. } =
                         &engines[0].columns[engines[0].field_index[&col_name]]
                     {
-                        dict.clone()
+                        (data.clone(), offsets.clone())
                     } else {
                         return engines_to_record_batches(engines, &plan);
                     };
-                let seed = crate::dict::SeedDict::new(first_dict);
+                // Build SeedDict from the contiguous buffer
+                let mut seed_values = Vec::new();
+                let mut seed_offsets = Vec::with_capacity(first_data.1.len());
+                seed_offsets.push(0);
+                for i in 0..(first_data.1.len() - 1) {
+                    let start = first_data.1[i] as usize;
+                    let end = first_data.1[i + 1] as usize;
+                    seed_values.extend_from_slice(&first_data.0[start..end]);
+                    seed_offsets.push(seed_values.len() as i32);
+                }
+                let seed = crate::dict::SeedDict {
+                    values: seed_values,
+                    offsets: seed_offsets,
+                    index: rustc_hash::FxHashMap::default(), // Will be populated by unify
+                };
                 let col_refs: Vec<&crate::columnar::ColumnBuilder> = engines
                     .iter()
                     .filter_map(|e| e.field_index.get(&col_name).map(|&idx| &e.columns[idx]))
                     .collect();
                 let (unified_dict, remaps) = crate::dict::unify_dictionaries(&seed, &col_refs);
-                // Build unified dict values and index for replace_dict.
-                let mut new_dict = Vec::with_capacity(unified_dict.offsets.len() - 1);
-                for w in unified_dict.offsets.windows(2) {
-                    let start = w[0] as usize;
-                    let end = w[1] as usize;
-                    let s = std::str::from_utf8(&unified_dict.data[start..end])
-                        .unwrap_or("")
-                        .to_owned();
-                    new_dict.push(s);
+                // Build unified data+offsets+index for replace_dict.
+                let mut new_index: rustc_hash::FxHashMap<Box<str>, i32> = rustc_hash::FxHashMap::default();
+                for i in 0..(unified_dict.offsets.len() - 1) {
+                    let start = unified_dict.offsets[i] as usize;
+                    let end = unified_dict.offsets[i + 1] as usize;
+                    let s = std::str::from_utf8(&unified_dict.data[start..end]).unwrap_or("");
+                    let k: Box<str> = s.into();
+                    new_index.insert(k, i as i32);
                 }
-                let new_index: HashMap<String, i32> = new_dict
-                    .iter()
-                    .enumerate()
-                    .map(|(i, s)| (s.clone(), i as i32))
-                    .collect();
                 // Apply remap to each chunk, then replace dict.
                 for (e, remap) in engines.iter_mut().zip(remaps.iter()) {
                     if let Some(idx) = e.field_index.get(&col_name).copied() {
                         if let Some(codes) = e.columns[idx].dict_codes_mut() {
                             codes.remap_codes(&remap.map);
                         }
-                        e.columns[idx].replace_dict(new_dict.clone(), new_index.clone());
+                        e.columns[idx].replace_dict(
+                            unified_dict.data.clone(),
+                            unified_dict.offsets.clone(),
+                            new_index.clone(),
+                        );
                     }
                 }
                 return engines_to_record_batches(engines, &plan);