nc-gcode-interpreter 0.2.8__tar.gz → 0.2.9__tar.gz

{nc_gcode_interpreter-0.2.8 → nc_gcode_interpreter-0.2.9}/.github/workflows/build-and-release.yml

@@ -156,7 +156,9 @@ jobs:
         uses: PyO3/maturin-action@v1
         with:
           target: ${{ matrix.platform.target }}
-          args: --release --out dist --find-interpreter
+          # Named interpreters: --find-interpreter also picks up the image's PyPy,
+          # which pyo3-ffi 0.29.0 fails to compile for.
+          args: --release --out dist -i python3.12 -i python3.13 -i python3.14 -i python3.14t
           sccache: 'true'
           manylinux: auto
       - name: Upload wheels

{nc_gcode_interpreter-0.2.8 → nc_gcode_interpreter-0.2.9}/CHANGELOG.md

@@ -4,6 +4,31 @@ Notable changes to **nc-gcode-interpreter**. The format loosely follows
 [Keep a Changelog](https://keepachangelog.com/); versions are git tags,
 released to PyPI.
 
+## [v0.2.9] - 2026-11-01
+
+### Changed
+
+- `nc_to_batches` and `nc_to_rows` borrow the Python str's UTF-8 buffer for
+  the run instead of copying it twice (once into a Rust `String`, once more
+  for error messages). On a 5M-line, 322 MB program the peak footprint of
+  `nc_to_batches` drops from 1.28 to 0.72 GiB. The Rust entry points
+  (`nc_to_table`, `nc_to_row_stream`, `nc_to_batch_stream*`) take
+  `impl Into<SourceText>`: a `&str` still works and is copied once, an owned
+  `String` or `Arc<str>` is shared without a copy (#89).
+
+- Every jump-free program now runs on the streaming line driver, not only
+  those of 64 MiB or more, and the streaming driver decodes ahead on a second
+  thread instead of decoding inline. On a 1M-line, 63 MB program
+  `nc_to_batches` peaks at 0.21 GiB instead of 0.60 GiB and takes 1.02 s
+  instead of 1.24 s; at 5M lines it takes 5.1 s instead of 6.3 s. Programs
+  with jumps keep the indexed driver; `NC_STREAM_MIN_BYTES` still sets a size
+  threshold (#89).
+
+- Less work per output row on the interpreter thread: the batch builder moves
+  string cells out of the rows it recycles instead of cloning them, and the
+  per-row column set and the flattener's position maps hash with FxHash. At
+  5M lines `nc_to_batches` takes 4.4 s instead of 5.2 s (#89).
+
 ## [v0.2.8] - 2026-10-15
 
 ### Added

{nc_gcode_interpreter-0.2.8 → nc_gcode_interpreter-0.2.9}/Cargo.lock

@@ -506,7 +506,7 @@ checksum = "88904434abc2901f197fe8cc55f0445e7ded921dba5911dad2e2b39b48e663c4"
 
 [[package]]
 name = "nc-gcode-interpreter"
-version = "0.2.8"
+version = "0.2.9"
 dependencies = [
  "arrow-array",
  "arrow-data",

{nc_gcode_interpreter-0.2.8 → nc_gcode_interpreter-0.2.9}/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "nc-gcode-interpreter"
-version = "0.2.8"
+version = "0.2.9"
 edition = "2021"
 readme = "README.md"
 

{nc_gcode_interpreter-0.2.8 → nc_gcode_interpreter-0.2.9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nc-gcode-interpreter
-Version: 0.2.8
+Version: 0.2.9
 Classifier: Programming Language :: Rust
 Classifier: Programming Language :: Python :: Implementation :: CPython
 Classifier: Programming Language :: Python :: 3.12

{nc_gcode_interpreter-0.2.8 → nc_gcode_interpreter-0.2.9}/README.md

@@ -161,18 +161,19 @@ for line_no, row, variables in nc_to_rows(program, include_variables=True):
 
 Rows are typed and forward-filled like the batch DataFrame (disable with `forward_fill=False`); errors raise from `next()` at the offending row; after exhaustion the iterator's `state` attribute holds the final interpreter state. See `python/example/streaming.py` for a runnable version.
 
-**Memory.** Programs of 64 MiB or more without multi-line control blocks or
-jumps are validated in a parallel preflight scan and then executed one line at a
-time, so parse storage is bounded by the longest line instead of the program
-length. The input text, its diagnostic copy and line-offset index, variable
-state and any output the caller retains still grow with the program, so use
-`nc_to_batches` with a bounded consumer for large files. Smaller programs and
-programs with jumps keep the indexed fast path; `NC_STREAM_MIN_BYTES` sets the
-threshold in bytes (0 streams every jump-free program). Control blocks and
-`NC_STAGE1=0` still use the whole-file parser. On a 1.28M-line (112 MB) field
-program the peak RSS of `nc_to_batches` drops from about 1.3 GiB to 0.4 GiB,
-and the second decode during execution costs up to about 20 percent of
-interpretation time. Measure a local program with
+**Memory.** Programs without multi-line control blocks or jumps are validated
+in a preflight scan and then executed one line at a time, with the next lines
+decoded on a second thread, so parse storage is bounded by a chunk of lines
+instead of the program length. The input text (held once: the Python str is
+borrowed, not copied, and error messages read from the same buffer), its
+line-offset index, variable state and any output the caller retains still grow
+with the program, so use `nc_to_batches` with a bounded consumer for large
+files. Programs with jumps keep the indexed fast path; `NC_STREAM_MIN_BYTES`
+sets a size below which jump-free programs take it too (default 0). Control
+blocks and `NC_STAGE1=0` still use the whole-file parser. On a 1M-line (63 MB)
+generated program the peak footprint of `nc_to_batches` is about 0.2 GiB, and
+streaming is 5 to 20 percent faster than the indexed driver from 1 MB up.
+Measure a local program with
 `python scripts/benchmark.py --input <file> --modes fresh --edits none
 --threads 4 --samples 1 --batch-size 4096 --allow-undefined --axis-index-e 4
 --axis-index-elx 5 --output tmp/memory.json`.

{nc_gcode_interpreter-0.2.8 → nc_gcode_interpreter-0.2.9}/src/flatten.rs

@@ -69,6 +69,7 @@
 use crate::errors::ParsingError;
 use crate::output::{intern_column, CellMap, Row, FLATTENED_COLUMN};
 use crate::state::emit_warning;
+use crate::state::FxMap;
 use crate::types::Value;
 use std::collections::HashMap;
 
@@ -136,7 +137,7 @@ pub struct Flattener {
     /// non-geometric value columns), interned.
     geometric_axes: Vec<&'static str>,
     /// Last known machine coordinate per geometric axis column.
-    positions: HashMap<&'static str, f64>,
+    positions: FxMap<&'static str, f64>,
     motion: Motion,
     plane: Plane,
     /// Buffered blocks of the active spline, in program order. Non-empty
@@ -144,7 +145,7 @@ pub struct Flattener {
     spline_buffer: Vec<SplineItem>,
     /// Snapshot of `positions` taken when the spline was selected: the
     /// spline's start point.
-    spline_start: HashMap<&'static str, f64>,
+    spline_start: FxMap<&'static str, f64>,
     /// Current B-spline degree (`SD=`, modal within the spline).
     spline_degree: usize,
     /// Curve motion words already warned about (CIP, CT, POLY, ...): one
@@ -179,11 +180,11 @@ impl Flattener {
         Ok(Flattener {
             tolerance,
             geometric_axes,
-            positions: HashMap::new(),
+            positions: FxMap::default(),
             motion: Motion::Linear,
             plane: G17,
             spline_buffer: Vec::new(),
-            spline_start: HashMap::new(),
+            spline_start: FxMap::default(),
             spline_degree: 3,
             warned_motions: Vec::new(),
         })
@@ -201,7 +202,7 @@ impl Flattener {
             }
             self.motion = new_motion;
         }
-        if let Some(Value::Str(plane)) = row.cells.get(intern_column("gg06_plane_select")) {
+        if let Some(Value::Str(plane)) = row.cells.get("gg06_plane_select") {
             self.plane = match plane.as_str() {
                 "G18" => G18,
                 "G19" => G19,
@@ -215,7 +216,7 @@ impl Flattener {
                     self.spline_start = self.positions.clone();
                     self.spline_degree = 3;
                 }
-                if let Some(Value::Float(sd)) = row.cells.get(intern_column("SD")) {
+                if let Some(Value::Float(sd)) = row.cells.get("SD") {
                     self.spline_degree = (*sd as usize).max(1);
                 }
                 let has_point = self
@@ -659,7 +660,7 @@ fn points_value(points: &[Vec<f64>], point_rows: &[Option<usize>], item_index: u
 }
 
 fn row_motion(row: &Row) -> Option<Motion> {
-    match row.cells.get(intern_column("gg01_motion")) {
+    match row.cells.get("gg01_motion") {
         Some(Value::Str(m)) => Some(match m.as_str() {
             "G0" | "G1" => Motion::Linear,
             "G2" => Motion::Arc { cw: true },

{nc_gcode_interpreter-0.2.8 → nc_gcode_interpreter-0.2.9}/src/interpreter.rs

@@ -6,6 +6,54 @@ use crate::state::{self, State};
 use crate::types::{NCParser, Rule};
 use pest::Parser;
 use std::collections::HashMap;
+use std::sync::Arc;
+
+/// Program text shared between the driver that interprets it and the state's
+/// diagnostics, so a run holds a single copy of the source. Converting from a
+/// `&str` copies once; an owned `String`, an `Arc<str>` or any other
+/// [`SourceText::new`] buffer is shared as is.
+#[derive(Clone)]
+pub struct SourceText(Arc<dyn AsRef<str> + Send + Sync>);
+
+impl SourceText {
+    pub fn new<T: AsRef<str> + Send + Sync + 'static>(text: T) -> Self {
+        SourceText(Arc::new(text))
+    }
+
+    pub fn as_str(&self) -> &str {
+        (*self.0).as_ref()
+    }
+}
+
+impl std::fmt::Debug for SourceText {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("SourceText").field("len", &self.as_str().len()).finish()
+    }
+}
+
+impl From<&str> for SourceText {
+    fn from(text: &str) -> Self {
+        SourceText(Arc::new(Box::<str>::from(text)))
+    }
+}
+
+impl From<&String> for SourceText {
+    fn from(text: &String) -> Self {
+        SourceText::from(text.as_str())
+    }
+}
+
+impl From<String> for SourceText {
+    fn from(text: String) -> Self {
+        SourceText::new(text)
+    }
+}
+
+impl From<Arc<str>> for SourceText {
+    fn from(text: Arc<str>) -> Self {
+        SourceText::new(text)
+    }
+}
 
 const DEFAULT_AXIS_IDENTIFIERS: &[&str] = &[
     "N", "X", "Y", "Z", "A", "B", "C", "D", "E", "F", "S", "U", "V", "RA1", "RA2", "RA3", "RA4", "RA5", "RA6",
@@ -13,7 +61,7 @@ const DEFAULT_AXIS_IDENTIFIERS: &[&str] = &[
 
 /// Main function: interpret the input program into a sanitized output table.
 pub fn nc_to_table(
-    input: &str,
+    input: impl Into<SourceText>,
     initial_state: Option<&str>,
     axis_identifiers: Option<Vec<String>>,
     extra_axes: Option<Vec<String>>,
@@ -41,7 +89,7 @@ pub fn nc_to_table(
     // Now interpret the main input using the axis_index_map from state
     let mut output = OutputRows::collect();
     install_flattener(&mut output, &state, flatten_tolerance)?;
-    interpret_file(input, &mut state, &mut output)?;
+    interpret_source(input.into(), &mut state, &mut output)?;
     let rows = output.finish()?;
 
     let table = Table::from_rows(&rows, disable_forward_fill);
@@ -104,7 +152,7 @@ fn install_flattener(
 #[allow(clippy::too_many_arguments)]
 #[allow(dead_code)] // used by the python-feature bindings, not the bin
 pub fn nc_to_row_stream(
-    input: &str,
+    input: impl Into<SourceText>,
     initial_state: Option<&str>,
     axis_identifiers: Option<Vec<String>>,
     extra_axes: Option<Vec<String>>,
@@ -127,7 +175,7 @@ pub fn nc_to_row_stream(
     }
     let mut output = OutputRows::stream(sender);
     install_flattener(&mut output, &state, flatten_tolerance)?;
-    interpret_file(input, &mut state, &mut output)?;
+    interpret_source(input.into(), &mut state, &mut output)?;
     output.finish()?;
     Ok(state)
 }
@@ -145,7 +193,7 @@ pub fn nc_to_row_stream(
 #[allow(dead_code)] // used by the python-feature bindings, not the bin
 #[allow(clippy::too_many_arguments)]
 pub fn nc_to_batch_stream(
-    input: &str,
+    input: impl Into<SourceText>,
     initial_state: Option<&str>,
     axis_identifiers: Option<Vec<String>>,
     extra_axes: Option<Vec<String>>,
@@ -183,7 +231,7 @@ pub fn nc_to_batch_stream(
 /// callers.
 #[allow(clippy::too_many_arguments)]
 pub fn nc_to_batch_stream_with_line_numbers(
-    input: &str,
+    input: impl Into<SourceText>,
     initial_state: Option<&str>,
     axis_identifiers: Option<Vec<String>>,
     extra_axes: Option<Vec<String>>,
@@ -218,15 +266,20 @@ pub fn nc_to_batch_stream_with_line_numbers(
         emit_line_no,
     );
     install_flattener(&mut output, &state, flatten_tolerance)?;
-    interpret_file(input, &mut state, &mut output)?;
+    interpret_source(input.into(), &mut state, &mut output)?;
     output.finish()?;
     Ok(state)
 }
 
 /// Interpret a file, pushing rows into `output`.
 fn interpret_file(input: &str, state: &mut State, output: &mut OutputRows) -> Result<(), ParsingError> {
-    // Store input for error messages
-    state.set_input(input);
+    interpret_source(SourceText::from(input), state, output)
+}
+
+/// As [`interpret_file`], sharing `source` with the state's diagnostics.
+fn interpret_source(source: SourceText, state: &mut State, output: &mut OutputRows) -> Result<(), ParsingError> {
+    state.set_source(source.clone());
+    let input = source.as_str();
 
     // Validate control-structure nesting first: a PEG reports an unclosed
     // IF/WHILE at the end of the file; the line scan reports the opener.
@@ -698,6 +751,32 @@ mod tests {
             .unwrap_or_else(|| panic!("column {name} missing; have {:?}", column_names(table)))
     }
 
+    /// An owned program is shared with the state's diagnostics, not copied.
+    #[test]
+    fn owned_source_is_shared_with_diagnostics() {
+        let program = String::from("G1 X0 Y0 F100\nX10 Y=\n");
+        let ptr = program.as_ptr();
+        let source = SourceText::from(program);
+        assert_eq!(source.as_str().as_ptr(), ptr);
+        let mut state = build_state(None, None, 10000, None, false);
+        state.set_source(source.clone());
+        assert_eq!(state.get_line(2), Some("X10 Y="));
+        assert_eq!(state.get_line(2).unwrap().as_ptr(), source.as_str()[14..].as_ptr());
+        let err = nc_to_table(
+            String::from("G1 X0\nX10 Y=\n"),
+            None,
+            None,
+            None,
+            10000,
+            false,
+            None,
+            false,
+            None,
+        )
+        .expect_err("a dangling assignment does not parse");
+        assert!(err.to_string().contains("X10 Y="), "{err}");
+    }
+
     /// `include_line_numbers` (default off) prepends a `line_no` column: one
     /// 1-based source line per output row, repeated under a loop and
     /// non-monotonic across a jump - matching what `nc_to_rows` yields.

{nc_gcode_interpreter-0.2.8 → nc_gcode_interpreter-0.2.9}/src/lib.rs

@@ -22,6 +22,7 @@ pub use line_driver::experiments;
 mod python_bindings {
     use pyo3::exceptions::PyValueError;
     use pyo3::prelude::*;
+    use pyo3::pybacked::PyBackedStr;
     use pyo3::types::PyCapsule;
     use pyo3::types::{PyDict, PyList};
     use pyo3::wrap_pyfunction;
@@ -64,7 +65,7 @@ mod python_bindings {
     use std::sync::{mpsc, Mutex};
 
     use crate::errors::ErrorLocation;
-    use crate::interpreter::{nc_to_batch_stream_with_line_numbers, nc_to_row_stream};
+    use crate::interpreter::{nc_to_batch_stream_with_line_numbers, nc_to_row_stream, SourceText};
     use crate::output::{is_forward_filled_column, is_string_column, Column, Row, Table, VariableEvents};
     use crate::state::FinalState;
     use crate::types::Value;
@@ -206,11 +207,23 @@ mod python_bindings {
         })
     }
 
+    /// The program for a run: the file at `input` when `input_is_path`, else
+    /// the Python str itself, shared without a copy.
+    fn source_text(input: PyBackedStr, input_is_path: bool) -> PyResult<SourceText> {
+        if input_is_path {
+            let text = std::fs::read_to_string(&*input)
+                .map_err(|e| PyErr::new::<PyValueError, _>(format!("Error reading input file '{}': {}", input, e)))?;
+            Ok(SourceText::from(text))
+        } else {
+            Ok(SourceText::new(input))
+        }
+    }
+
     /// Spawn the interpreter on a worker thread, pushing finished rows into a
     /// bounded channel. Shared by the row and batch streaming iterators.
     #[allow(clippy::too_many_arguments)]
     fn spawn_stream(
-        input: String,
+        input: SourceText,
         initial_state: Option<String>,
         axis_identifiers: Option<Vec<String>>,
         extra_axes: Option<Vec<String>>,
@@ -230,7 +243,7 @@ mod python_bindings {
             .name("nc-interpreter".to_string())
             .spawn(move || {
                 let outcome = nc_to_row_stream(
-                    &input,
+                    input,
                     initial_state.as_deref(),
                     axis_identifiers,
                     extra_axes,
@@ -258,7 +271,7 @@ mod python_bindings {
     /// another is consumed). Used by `nc_to_batches`.
     #[allow(clippy::too_many_arguments)]
     fn spawn_batch_stream(
-        input: String,
+        input: SourceText,
         batch_size: usize,
         initial_state: Option<String>,
         axis_identifiers: Option<Vec<String>>,
@@ -290,7 +303,7 @@ mod python_bindings {
             .name("nc-interpreter".to_string())
             .spawn(move || {
                 let outcome = nc_to_batch_stream_with_line_numbers(
-                    &input,
+                    input,
                     initial_state.as_deref(),
                     axis_identifiers,
                     extra_axes,
@@ -491,7 +504,7 @@ mod python_bindings {
     #[pyo3(signature = (input, initial_state = None, axis_identifiers = None, extra_axes = None, iteration_limit = 10000, forward_fill = true, include_variables = false, axis_index_map = None, allow_undefined_variables = false, input_is_path = false, flatten_tolerance = None))]
     #[allow(clippy::too_many_arguments)]
     fn nc_to_rows(
-        input: String,
+        input: PyBackedStr,
         initial_state: Option<String>,
         axis_identifiers: Option<Vec<String>>,
         extra_axes: Option<Vec<String>>,
@@ -504,15 +517,10 @@ mod python_bindings {
         flatten_tolerance: Option<f64>,
     ) -> PyResult<NcRowIterator> {
         // When `input_is_path` is set, `input` is a filesystem path: read the
-        // program here (once) instead of copying a 1.1 GB Python str across the
-        // PyO3 boundary. Reading before the worker spawns surfaces a missing
-        // file as an immediate, clean error.
-        let input = if input_is_path {
-            std::fs::read_to_string(&input)
-                .map_err(|e| PyErr::new::<PyValueError, _>(format!("Error reading input file '{}': {}", input, e)))?
-        } else {
-            input
-        };
+        // program here, before the worker spawns, so a missing file surfaces
+        // as an immediate, clean error. Otherwise the worker borrows the
+        // Python str's UTF-8 buffer for the run instead of copying it.
+        let input = source_text(input, input_is_path)?;
         let (row_receiver, result_receiver, handle) = spawn_stream(
             input,
             initial_state,
@@ -668,7 +676,7 @@ mod python_bindings {
     #[pyo3(signature = (input, batch_size = 500_000, initial_state = None, axis_identifiers = None, extra_axes = None, iteration_limit = 10000, disable_forward_fill = false, axis_index_map = None, allow_undefined_variables = false, input_is_path = false, flatten_tolerance = None, include_line_numbers = false, include_variables = false))]
     #[allow(clippy::too_many_arguments)]
     fn nc_to_batches(
-        input: String,
+        input: PyBackedStr,
         batch_size: usize,
         initial_state: Option<String>,
         axis_identifiers: Option<Vec<String>>,
@@ -686,12 +694,7 @@ mod python_bindings {
             return Err(PyErr::new::<PyValueError, _>("batch_size must be greater than 0"));
         }
         // Read the program here when given a path (see nc_to_rows).
-        let input = if input_is_path {
-            std::fs::read_to_string(&input)
-                .map_err(|e| PyErr::new::<PyValueError, _>(format!("Error reading input file '{}': {}", input, e)))?
-        } else {
-            input
-        };
+        let input = source_text(input, input_is_path)?;
         let (batch_receiver, result_receiver, events_receiver, handle) = spawn_batch_stream(
             input,
             batch_size,

{nc_gcode_interpreter-0.2.8 → nc_gcode_interpreter-0.2.9}/src/line_driver.rs

@@ -43,10 +43,16 @@ pub mod experiments;
 /// of the test suite, most hand-written files) stay single-threaded.
 const PARALLEL_MIN_BYTES: usize = 4 << 20; // 4 MiB
 
-/// Below this many bytes the indexed driver runs jump-free programs: its
-/// retained decode state is small there, and it skips the streaming driver's
-/// second decode. NC_STREAM_MIN_BYTES overrides it (0 streams every program).
-const STREAMING_MIN_BYTES: usize = 64 << 20; // 64 MiB
+/// Below this many bytes the indexed driver runs jump-free programs. The
+/// streaming driver decodes on a second thread, so it is as fast from about
+/// 1 MiB up and within 0.3 ms below, and it retains no per-line state: every
+/// jump-free program streams. NC_STREAM_MIN_BYTES overrides it (a huge value
+/// pins the indexed driver).
+const STREAMING_MIN_BYTES: usize = 0;
+
+/// Below this many bytes the streaming driver's preflight and decode stay on
+/// the calling thread; the thread hand-off costs more than it saves there.
+const STREAM_PARALLEL_MIN_BYTES: usize = 64 << 10; // 64 KiB
 
 type Output = crate::output::OutputRows;
 
@@ -266,7 +272,7 @@ pub fn interpret_lines(input: &str, output: &mut Output, state: &mut State) -> R
 /// holds a jump decides, exactly as a sequential scan would.
 fn preflight_streaming(input: &str, state: &State) -> Result<bool, ParsingError> {
     let threads = rayon::current_num_threads().max(1);
-    let parts = if input.len() < PARALLEL_MIN_BYTES || threads <= 1 {
+    let parts = if input.len() < STREAM_PARALLEL_MIN_BYTES || threads <= 1 {
         1
     } else {
         threads * 8
@@ -380,23 +386,88 @@ fn chunk_at_newlines(input: &str, parts: usize) -> Vec<Range<usize>> {
     ranges
 }
 
-/// O(longest line) decoding/parse storage, independent of the number of lines.
+/// O(chunk) decoding/parse storage, independent of the number of lines.
 /// Input ownership, diagnostic line offsets and the selected output sink are
 /// separate: collecting output still necessarily grows with the program.
 fn interpret_streaming(input: &str, output: &mut Output, state: &mut State) -> Result<BlockFlow, ParsingError> {
-    let mut arena = Vec::new();
-    for (index, line) in input.lines().enumerate() {
-        arena.clear();
-        let flow = match decode_line(line, index + 1, &mut arena) {
-            DecodeResult::Blank => continue,
-            DecodeResult::Trivial(decoded, _) => execute_decoded(&decoded, &arena, output, state)?,
-            DecodeResult::NeedsGrammar => interpret_block(parse_single_line(line, index, state)?, output, state)?,
+    if input.len() < STREAM_PARALLEL_MIN_BYTES || rayon::current_num_threads() <= 1 {
+        let mut arena = Vec::new();
+        for (index, line) in input.lines().enumerate() {
+            arena.clear();
+            let decoded = decode_line(line, index + 1, &mut arena);
+            let flow = execute_streamed(&decoded, line, index, &arena, output, state)?;
+            if !matches!(flow, BlockFlow::Continue) {
+                return Ok(flow);
+            }
+        }
+        return Ok(BlockFlow::Continue);
+    }
+    // Decode on a second thread, a few chunks ahead, so execution does not pay
+    // for the decode the indexed driver runs in parallel.
+    std::thread::scope(|scope| {
+        let (sender, receiver) = std::sync::mpsc::sync_channel::<DecodedChunk>(2);
+        scope.spawn(move || decode_chunks(input, sender));
+        // Returning early drops `receiver`, which stops the decode thread.
+        for chunk in receiver {
+            for (offset, (decoded, line)) in chunk.results.iter().zip(&chunk.lines).enumerate() {
+                let flow = execute_streamed(decoded, line, chunk.first_line + offset, &chunk.arena, output, state)?;
+                if !matches!(flow, BlockFlow::Continue) {
+                    return Ok(flow);
+                }
+            }
+        }
+        Ok(BlockFlow::Continue)
+    })
+}
+
+/// Lines per chunk the streaming decode thread hands to execution.
+const STREAM_CHUNK_LINES: usize = 4096;
+
+/// A run of consecutive decoded lines; `first_line` is the 0-based index of
+/// the first one, and `results` index into `arena` like the indexed driver's.
+struct DecodedChunk<'a> {
+    first_line: usize,
+    lines: Vec<&'a str>,
+    results: Vec<DecodeResult<'a>>,
+    arena: Vec<Word<'a>>,
+}
+
+fn decode_chunks<'a>(input: &'a str, sender: std::sync::mpsc::SyncSender<DecodedChunk<'a>>) {
+    let mut lines = input.lines().peekable();
+    let mut first_line = 0;
+    while lines.peek().is_some() {
+        let mut chunk = DecodedChunk {
+            first_line,
+            lines: lines.by_ref().take(STREAM_CHUNK_LINES).collect(),
+            results: Vec::with_capacity(STREAM_CHUNK_LINES),
+            arena: Vec::with_capacity(STREAM_CHUNK_LINES * 4),
         };
-        if !matches!(flow, BlockFlow::Continue) {
-            return Ok(flow);
+        for (offset, line) in chunk.lines.iter().enumerate() {
+            chunk
+                .results
+                .push(decode_line(line, first_line + offset + 1, &mut chunk.arena));
+        }
+        first_line += chunk.lines.len();
+        if sender.send(chunk).is_err() {
+            return;
         }
     }
-    Ok(BlockFlow::Continue)
+}
+
+/// Execute one streamed line (0-based `index`) from its decode result.
+fn execute_streamed(
+    decoded: &DecodeResult,
+    line: &str,
+    index: usize,
+    arena: &[Word],
+    output: &mut Output,
+    state: &mut State,
+) -> Result<BlockFlow, ParsingError> {
+    match decoded {
+        DecodeResult::Blank => Ok(BlockFlow::Continue),
+        DecodeResult::Trivial(decoded, _) => execute_decoded(decoded, arena, output, state),
+        DecodeResult::NeedsGrammar => interpret_block(parse_single_line(line, index, state)?, output, state),
+    }
 }
 
 fn parse_single_line<'i>(line: &'i str, offset: usize, state: &State) -> Result<Pair<'i, Rule>, ParsingError> {
@@ -1644,6 +1715,38 @@ M30
         assert!(rx.try_recv().is_err(), "syntax failure must not emit earlier rows");
     }
 
+    /// Programs past STREAM_PARALLEL_MIN_BYTES and several decode chunks long
+    /// take the pipelined decode; output, M30 and late runtime errors agree.
+    #[test]
+    fn pipelined_streaming_matches_full_parse() {
+        use std::fmt::Write;
+        let mut body = String::from("MULTIPLIER=2\nR1=0\n");
+        for i in 0..3 * super::STREAM_CHUNK_LINES {
+            if i % 997 == 0 {
+                body.push_str("X=SIN(30) R1=R1+1\n");
+            } else {
+                writeln!(body, "N{i} G1 X{}.5 Y{} E=IC(MULTIPLIER*0.01) ; c", i % 100, i % 7).unwrap();
+            }
+        }
+        assert!(body.len() > super::STREAM_PARALLEL_MIN_BYTES);
+        let mid = body.len() / 2 + body[body.len() / 2..].find('\n').unwrap() + 1;
+        for program in [
+            body.clone(),
+            format!("{}M30\n{}", &body[..mid], &body[mid..]),
+            format!("{}X=SIN(MISSING)\n{}", &body[..mid], &body[mid..]),
+        ] {
+            assert_paths_agree(&program, false);
+        }
+        // The shared error check compares only the line text; pin the number.
+        let program = format!("{}X=SIN(MISSING)\n{}", &body[..mid], &body[mid..]);
+        let _guard = env_lock();
+        std::env::set_var("NC_STREAM_MIN_BYTES", "0");
+        let error = run(&program, true, false).unwrap_err();
+        std::env::remove_var("NC_STREAM_MIN_BYTES");
+        let expected = body[..mid].lines().count() + 1;
+        assert_eq!(error.location().map(|l| l.line), Some(expected), "{error}");
+    }
+
     #[test]
     fn preflight_chunks_agree_with_a_single_chunk() {
         let mut state = crate::state::State::new(vec!["X".into()], 100, None, false);

{nc_gcode_interpreter-0.2.8 → nc_gcode_interpreter-0.2.9}/src/main.rs

@@ -142,7 +142,7 @@ fn main() -> io::Result<()> {
     let flatten_tolerance = matches.get_one::<f64>("flatten_tolerance").copied();
 
     match nc_to_table(
-        &input,
+        input,
         initial_state.as_deref(),
         axes_override.clone(),
         extra_axes,