nc-gcode-interpreter 0.2.4__tar.gz → 0.2.5__tar.gz

{nc_gcode_interpreter-0.2.4 → nc_gcode_interpreter-0.2.5}/CHANGELOG.md

@@ -4,6 +4,20 @@ Notable changes to **nc-gcode-interpreter**. The format loosely follows
 [Keep a Changelog](https://keepachangelog.com/); versions are git tags,
 released to PyPI.
 
+## [v0.2.5] - 2026-07-08
+
+### Changed
+
+- Interpreter throughput on the DataFrame/batch path: the per-output-row `Row`
+  allocations are now recycled instead of freed. Profiling the 1.1 GB → DataFrame
+  conversion showed ~50% of CPU in the system allocator, dominated by allocating
+  and freeing each of the 22M rows' cell buffers. After a batch is built its rows
+  are cleared (capacity retained) into a pool and handed back to the interpreter
+  to refill, bounding live row allocations to ~2× the batch size instead of the
+  whole-file row count. Output is byte-identical; the streaming (`nc_to_rows`)
+  and in-memory collect paths are unchanged. ~10% off end-to-end on the large
+  real-world program (#61).
+
 ## [v0.2.4] - 2026-07-08
 
 ### Changed

{nc_gcode_interpreter-0.2.4 → nc_gcode_interpreter-0.2.5}/Cargo.lock

@@ -506,7 +506,7 @@ checksum = "88904434abc2901f197fe8cc55f0445e7ded921dba5911dad2e2b39b48e663c4"
 
 [[package]]
 name = "nc-gcode-interpreter"
-version = "0.2.4"
+version = "0.2.5"
 dependencies = [
  "arrow-array",
  "arrow-data",

{nc_gcode_interpreter-0.2.4 → nc_gcode_interpreter-0.2.5}/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "nc-gcode-interpreter"
-version = "0.2.4"
+version = "0.2.5"
 edition = "2021"
 readme = "README.md"
 

{nc_gcode_interpreter-0.2.4 → nc_gcode_interpreter-0.2.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nc-gcode-interpreter
-Version: 0.2.4
+Version: 0.2.5
 Classifier: Programming Language :: Rust
 Classifier: Programming Language :: Python :: Implementation :: CPython
 Classifier: Programming Language :: Python :: 3.12

{nc_gcode_interpreter-0.2.4 → nc_gcode_interpreter-0.2.5}/src/output.rs

@@ -82,6 +82,13 @@ impl CellMap {
         None
     }
 
+    /// Empty the cells, retaining the backing allocation so a recycled row can
+    /// be refilled without a fresh heap allocation (see [`Row::reset_for_reuse`]).
+    #[inline]
+    pub fn clear(&mut self) {
+        self.entries.clear();
+    }
+
     #[inline]
     pub fn get_mut(&mut self, key: &str) -> Option<&mut Value> {
         self.entries.iter_mut().find(|(k, _)| *k == key).map(|(_, v)| v)
@@ -131,6 +138,20 @@ pub struct Row {
     pub variable_changes: Vec<(String, f64)>,
 }
 
+impl Row {
+    /// Empty this row for reuse as the next `current`, retaining both the cells
+    /// and variable-change allocations. On the batch path a finished row's
+    /// buffers are consumed into the columnar batch and then recycled here,
+    /// which collapses the per-row heap alloc/free (~22M on the 1.1 GB file,
+    /// the dominant allocator cost) into a small pool cycled batch-to-batch.
+    #[inline]
+    pub fn reset_for_reuse(&mut self) {
+        self.line_no = 0;
+        self.cells.clear();
+        self.variable_changes.clear();
+    }
+}
+
 /// Sparse side-table of variable-change events for the batch path, mirroring
 /// what the streaming `nc_to_rows` yields per row as `variable_changes`. Stored
 /// columnar (struct-of-arrays) so it converts to an Arrow table cheaply:
@@ -205,6 +226,10 @@ pub struct BatchStreamSink {
     sender: std::sync::mpsc::SyncSender<Table>,
     builder: BatchBuilder,
     buffer: Vec<Row>,
+    /// Emptied rows kept for reuse: after a batch is built the buffered rows'
+    /// allocations are cleared (not freed) into this pool, and `accept` hands
+    /// them back to the interpreter to refill instead of allocating fresh.
+    recycle: Vec<Row>,
     batch_size: usize,
     /// Whether to accumulate variable-change events (opt-in `include_variables`).
     record_variables: bool,
@@ -226,7 +251,7 @@ impl BatchStreamSink {
     /// rows have accumulated. Variable-only rows carry no output cells and are
     /// dropped from the batch table, but their variable changes are still
     /// recorded into the sparse event log when `record_variables` is on.
-    fn accept(&mut self, row: Row) -> Result<(), ParsingError> {
+    fn accept(&mut self, mut row: Row) -> Result<Option<Row>, ParsingError> {
         if self.record_variables {
             for (name, value) in &row.variable_changes {
                 let id = match self.name_ids.get(name) {
@@ -246,15 +271,20 @@ impl BatchStreamSink {
                 self.events.value.push(*value);
             }
         }
+        // A variable-only row carries no output cell: it is not buffered, so hand
+        // its (cleared) allocation straight back for reuse.
         if row.cells.is_empty() {
-            return Ok(());
+            row.reset_for_reuse();
+            return Ok(Some(row));
         }
         self.output_row_count += 1;
         self.buffer.push(row);
         if self.buffer.len() >= self.batch_size {
             self.emit()?;
         }
-        Ok(())
+        // Hand back a pooled row (refilled after the last emit) so the caller
+        // refills a retained allocation instead of allocating a fresh one.
+        Ok(self.recycle.pop())
     }
 
     /// Send the accumulated variable-change events once, at finish. A no-op when
@@ -272,8 +302,20 @@ impl BatchStreamSink {
         if self.buffer.is_empty() {
             return Ok(());
         }
-        let rows = std::mem::take(&mut self.buffer);
+        // Take the buffered rows out to build the batch, then recycle their
+        // allocations: `build_batch` only borrows the rows (it copies each cell
+        // value into the columnar builders), so once it returns the row buffers
+        // are free to be cleared and pooled rather than dropped.
+        let mut rows = std::mem::take(&mut self.buffer);
         let table = self.builder.build_batch(&rows);
+        self.recycle.reserve(rows.len());
+        for mut row in rows.drain(..) {
+            row.reset_for_reuse();
+            self.recycle.push(row);
+        }
+        // `rows` is now empty but keeps its `Vec<Row>` capacity - reinstate it as
+        // the buffer so the outer vector is not reallocated each batch either.
+        self.buffer = rows;
         self.sender.send(table).map_err(|_| ParsingError::StreamClosed)
     }
 }
@@ -362,6 +404,7 @@ impl OutputRows {
                 sender,
                 builder: BatchBuilder::new(disable_forward_fill).with_line_numbers(emit_line_no),
                 buffer: Vec::new(),
+                recycle: Vec::new(),
                 batch_size,
                 record_variables,
                 events_sender,
@@ -416,32 +459,53 @@ impl OutputRows {
         }
     }
 
-    /// Route a finished row to the sink, passing it through the flattener
-    /// first when one is installed. Shared by `flush`.
-    fn deliver(&mut self, row: Row) -> Result<(), ParsingError> {
+    /// Return a recycled row to the sink's pool. Used when a single `deliver`
+    /// yields more recycled rows than the one the caller can reuse as `current`
+    /// (the flattener fans one block out to many rows): the surplus goes back to
+    /// the pool instead of being freed, so recycling stays effective on
+    /// flattened paths too. A no-op for the non-recycling sinks.
+    fn reclaim(&mut self, row: Row) {
+        if let RowSink::Batch(sink) = &mut self.sink {
+            sink.recycle.push(row);
+        }
+    }
+
+    /// Route a row to the sink, passing it through the flattener first when one
+    /// is installed, and returning an emptied row the sink no longer needs
+    /// (batch path) so the caller can reuse its allocation as the next
+    /// `current`. `None` when nothing is available to recycle.
+    fn deliver(&mut self, row: Row) -> Result<Option<Row>, ParsingError> {
         if let Some(mut flattener) = self.flattener.take() {
             let mut flattened = Vec::new();
             flattener.push(row, &mut flattened);
             self.flattener = Some(flattener);
+            // Keep one recycled row for the caller to reuse; hand any others the
+            // sink returns this call straight back to the pool so they are not
+            // freed (a flattened block delivers many rows in one `deliver`).
+            let mut recycled = None;
             for row in flattened {
-                self.deliver_to_sink(row)?;
+                if let Some(r) = self.deliver_to_sink(row)? {
+                    if let Some(prev) = recycled.replace(r) {
+                        self.reclaim(prev);
+                    }
+                }
             }
-            return Ok(());
+            return Ok(recycled);
         }
         self.deliver_to_sink(row)
     }
 
     /// Route a finished row to the sink: collected, streamed row-at-a-time, or
     /// fed to the worker-side batch producer.
-    fn deliver_to_sink(&mut self, row: Row) -> Result<(), ParsingError> {
+    fn deliver_to_sink(&mut self, row: Row) -> Result<Option<Row>, ParsingError> {
         match &mut self.sink {
             RowSink::Collect(rows) => {
                 rows.push(row);
-                Ok(())
+                Ok(None)
             }
             // The receiver hung up: the consumer stopped iterating. Abort
             // interpretation instead of running the rest of the program.
-            RowSink::Stream(sender) => sender.send(row).map_err(|_| ParsingError::StreamClosed),
+            RowSink::Stream(sender) => sender.send(row).map(|_| None).map_err(|_| ParsingError::StreamClosed),
             RowSink::Batch(sink) => sink.accept(row),
         }
     }
@@ -464,7 +528,13 @@ impl OutputRows {
                 self.warned_g91 = true;
             }
         }
-        self.deliver(row)
+        // `mem::take` above left `current` as an empty (unallocated) default; if
+        // the sink handed back a pooled row, adopt its retained allocation for
+        // the next block instead of allocating fresh on the first cell insert.
+        if let Some(recycled) = self.deliver(row)? {
+            self.current = recycled;
+        }
+        Ok(())
     }
 
     /// Begin the row for the block at `line_no`, flushing the previous row.