abnf-rust 2.8.2__tar.gz → 2.9.0__tar.gz

{abnf_rust-2.8.2 → abnf_rust-2.9.0}/Cargo.lock

@@ -4,7 +4,7 @@ version = 3
 
 [[package]]
 name = "abnf-core"
-version = "2.8.2"
+version = "2.9.0"
 dependencies = [
  "caseless",
  "lru",
@@ -14,7 +14,7 @@ dependencies = [
 
 [[package]]
 name = "abnf-rust-ext"
-version = "2.8.2"
+version = "2.9.0"
 dependencies = [
  "abnf-core",
  "mimalloc",
@@ -100,9 +100,9 @@ dependencies = [
 
 [[package]]
 name = "lru"
-version = "0.18.1"
+version = "0.18.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0b6180140927ee907000b0aa540091f6ea512ead4447c92b8fc35bc72788a5a6"
+checksum = "5d2f2f9b4ba7e6b24d95e7e899329d35be83bcded72c8540cdd5368932d1d90a"
 dependencies = [
  "hashbrown",
 ]

{abnf_rust-2.8.2 → abnf_rust-2.9.0}/Cargo.toml

@@ -6,7 +6,7 @@ resolver = "2"
 # Lock-stepped with the `abnf` distribution; both packages release
 # under the same git tag so the wheels at `pip install abnf==X` and
 # `pip install abnf-rust==X` always come from the same source state.
-version = "2.8.2"
+version = "2.9.0"
 edition = "2021"
 rust-version = "1.75"
 license = "MIT"
@@ -14,7 +14,7 @@ authors = ["Charles Yeomans <charles@declaresub.com>"]
 repository = "https://github.com/declaresub/abnf"
 
 [workspace.dependencies]
-abnf-core = { path = "rust/core", version = "2.8.2" }
+abnf-core = { path = "rust/core", version = "2.9.0" }
 caseless = "0.2"
 lru = "0.18"
 mimalloc = { version = "0.1", default-features = false }

{abnf_rust-2.8.2 → abnf_rust-2.9.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: abnf-rust
-Version: 2.8.2
+Version: 2.9.0
 Classifier: Development Status :: 4 - Beta
 Classifier: Intended Audience :: Developers
 Classifier: License :: OSI Approved :: MIT License

{abnf_rust-2.8.2 → abnf_rust-2.9.0}/rust/core/src/lib.rs

@@ -47,7 +47,7 @@ pub use error::ParseError;
 pub use literal::{Literal, LiteralKind};
 pub use matcher::Match;
 pub use meta_grammar::{build_meta_grammar, install_meta_grammar};
-pub use node::{LiteralNode, Node, NodeKind};
+pub use node::{ForeignNode, LiteralNode, Node, NodeKind};
 pub use option::OptionParser;
 pub use parser::{arc, ArcParser, ExternalParser, MatchList, NodeList, ParseResult, Parser, Src};
 pub use prose::Prose;

{abnf_rust-2.8.2 → abnf_rust-2.9.0}/rust/core/src/literal.rs

@@ -126,14 +126,17 @@ impl Literal {
                 }
             }
             LiteralKind::String { value, pattern } => {
-                // Mirror Python's `if start < len(source)` guard (see
-                // `_lparse_value`), which raises even for an empty
-                // literal at EOF.  Without it the length check below
-                // would let a zero-length pattern match at any
-                // out-of-range start.
-                if start >= source.len() {
-                    return Err(self.parse_error(start));
-                }
+                // Enough source must remain for the whole literal.  This
+                // one check covers both cases: a non-empty literal needs
+                // room, and a zero-length one needs only `start <=
+                // source.len()`, which is what `end > source.len()` says
+                // when `plen` is 0.
+                //
+                // There used to be a `start >= source.len()` guard above
+                // this, mirroring Python's, which also refused a
+                // zero-length literal at end of input -- so `""` matched
+                // everywhere except there.  See
+                // https://github.com/declaresub/abnf/issues/260 .
                 let plen = value.len();
                 let end = start + plen;
                 if end > source.len() {
@@ -210,11 +213,22 @@ mod tests {
         assert!(lit.lparse(&[0xD800, 0x62], 0).is_err());
     }
 
+    /// #260: a zero-length literal matches the empty string wherever the
+    /// source reaches, end of input included -- but not past it.
     #[test]
-    fn empty_literal_raises_at_eof() {
+    fn empty_literal_matches_at_eof_but_not_beyond() {
         let lit = Literal::string("", false);
-        assert!(lit.lparse(&[], 0).is_err());
-        assert!(lit.lparse(&cps("x"), 0).is_ok());
+        assert!(lit.lparse(&[], 0).is_ok(), "empty source, offset 0");
+        assert!(lit.lparse(&cps("x"), 0).is_ok(), "start of input");
+        assert!(lit.lparse(&cps("x"), 1).is_ok(), "end of input");
+        assert!(lit.lparse(&cps("x"), 2).is_err(), "past end of input");
+        assert!(lit.lparse(&[], 1).is_err(), "past end of empty source");
+
+        // A non-empty literal still needs room for all of itself.
+        let lit = Literal::string("ab", false);
+        assert!(lit.lparse(&cps("ab"), 0).is_ok());
+        assert!(lit.lparse(&cps("a"), 0).is_err());
+        assert!(lit.lparse(&cps("ab"), 2).is_err());
     }
 
     #[test]

{abnf_rust-2.8.2 → abnf_rust-2.9.0}/rust/core/src/node.rs

@@ -75,11 +75,45 @@ impl LiteralNode {
     }
 }
 
+/// A terminal node supplied by an embedded Python parser, carrying the
+/// text that parser reported.
+///
+/// Engine-built terminals are spans of the source ([`LiteralNode`]),
+/// which is what lets the PyO3 layer produce their values by slicing.
+/// A node handed *in* by a `PyCallbackParser` need not correspond to
+/// any span: a custom parser may legitimately return a normalised or
+/// synthesised value, and the pure-Python backend keeps it.  Slicing
+/// the source for one of these replaced it with unrelated text
+/// (issue #220), so these carry their own.
+///
+/// Code points rather than a Rust string, for the same reason the
+/// source is: the value may contain a lone surrogate.
+#[derive(Debug, Clone)]
+pub struct ForeignNode {
+    pub value: Box<[u32]>,
+    /// Offset the parser reported.  Not necessarily where `value`
+    /// appears in the source -- it need not appear there at all.
+    pub offset: usize,
+    pub length: usize,
+}
+
+impl ForeignNode {
+    pub fn new(value: Box<[u32]>, offset: usize, length: usize) -> Self {
+        Self { value, offset, length }
+    }
+}
+
 /// Sum type for parse-tree children.
 #[derive(Debug, Clone)]
 pub enum NodeKind {
     Internal(Node),
     Literal(LiteralNode),
+    /// Behind an `Arc`, as `Node`'s children are, and for the same
+    /// reason: node lists are cloned on every match extension, so this
+    /// variant's clone lands on every parse whether or not a grammar
+    /// ever produces one.  Inline it also made `NodeKind` 40 bytes
+    /// rather than 32.
+    Foreign(Arc<ForeignNode>),
 }
 
 impl NodeKind {
@@ -98,6 +132,13 @@ impl NodeKind {
                         .map(|cp| char::from_u32(*cp).unwrap_or(char::REPLACEMENT_CHARACTER)),
                 );
             }
+            NodeKind::Foreign(f) => {
+                out.extend(
+                    f.value
+                        .iter()
+                        .map(|cp| char::from_u32(*cp).unwrap_or(char::REPLACEMENT_CHARACTER)),
+                );
+            }
         }
     }
 
@@ -113,6 +154,11 @@ impl NodeKind {
     pub fn span_bounds(&self) -> Option<(usize, usize)> {
         match self {
             NodeKind::Literal(l) => Some((l.offset, l.offset + l.length)),
+            // A foreign node's text is its own, so its span says where
+            // the parser claims it sat, not where its value can be
+            // read from.  Callers computing a value from a span must
+            // check `contains_foreign` first.
+            NodeKind::Foreign(f) => Some((f.offset, f.offset + f.length)),
             NodeKind::Internal(n) => n.children.iter().fold(None, |acc, c| {
                 match (acc, c.span_bounds()) {
                     (None, s) => s,
@@ -123,3 +169,4 @@ impl NodeKind {
         }
     }
 }
+

{abnf_rust-2.8.2 → abnf_rust-2.9.0}/rust/core/src/rule.rs

@@ -268,7 +268,17 @@ impl NamedRule {
                 self.name
             ),
         });
-        let def = self.definition().ok_or_else(|| self.parse_error(start))?;
+        // A rule that was never defined is a broken grammar, not input
+        // that failed to match.  Returning a `ParseError` here would be
+        // indistinguishable from "this alternative did not match", so
+        // an enclosing `Alternation` or `Repetition` would swallow it
+        // as backtracking and a typo'd rule name would silently delete
+        // that branch of the grammar.  The pure-Python backend raises
+        // `GrammarError`; panic with the phrase the PyO3 layer maps to
+        // it, the same route `is_excluded` and the recursion guard use.
+        let Some(def) = self.definition() else {
+            panic!("Undefined rule \"{}\"", self.name);
+        };
         let inner = def.lparse(source, start)?;
 
         let excluded = self.exclude();
@@ -296,9 +306,26 @@ impl NamedRule {
             if let Some(ref excluded) = excluded {
                 // `m.start` is the end of the match; the span is what
                 // the Python side rebuilds by joining node values.
-                if Self::is_excluded(excluded, &source[start..m.start]) {
-                    seen.remove(&m.start);
-                    continue;
+                //
+                // It is not necessarily a real span.  When this rule's
+                // definition is a `PyCallbackParser`, the offset comes
+                // from a Python parser and can be anything -- past the
+                // end of the source, or before `start`.  Slicing on it
+                // panicked, and a panic crosses the FFI as
+                // `PanicException`, a `BaseException` that `except
+                // ParseError` (or even `except Exception`) does not
+                // catch (issue #218).
+                //
+                // Text that is not in the source cannot be text the
+                // excluded rule matches, so a nonsensical span means
+                // "not excluded".  The bad offset then fails naturally
+                // further up, which is what the pure-Python backend
+                // does with the same input.
+                if let Some(span) = source.get(start..m.start) {
+                    if Self::is_excluded(excluded, span) {
+                        seen.remove(&m.start);
+                        continue;
+                    }
                 }
             }
             let node = Node::new(self.name.clone(), m.nodes.into_vec());

{abnf_rust-2.8.2 → abnf_rust-2.9.0}/rust/core/src/visitor.rs

@@ -153,6 +153,9 @@ fn visit_repeat(node: &Node, src: Src<'_>) -> Repeat {
                     saw_star = true;
                 }
             }
+            // The meta-grammar builds its own trees, so no node here
+            // came from an embedded Python parser.
+            NodeKind::Foreign(_) => {}
         }
     }
     let min = if min_src.is_empty() {

{abnf_rust-2.8.2 → abnf_rust-2.9.0}/rust/pyo3/src/nodes.rs

@@ -25,15 +25,15 @@ use pyo3::prelude::*;
 use pyo3::types::PyString;
 use pyo3::types::PyList;
 
-use abnf_core::{LiteralNode, Match, Node, NodeKind};
+use abnf_core::{ForeignNode, LiteralNode, Match, Node, NodeKind};
 
-use crate::source::substring;
+use crate::source::{from_code_points, substring, CodePoints};
 
 // ----------------------------------------------------------------
 // LiteralNode
 // ----------------------------------------------------------------
 
-#[pyclass(name = "LiteralNode", module = "abnf_rust._ext", from_py_object)]
+#[pyclass(name = "LiteralNode", module = "abnf_rust._ext", from_py_object, subclass)]
 #[derive(Debug)]
 pub struct PyLiteralNode {
     /// The matched text, as a Python `str` sliced out of the source.
@@ -94,13 +94,16 @@ impl PyLiteralNode {
             && self.value.bind(py).as_any().eq(other.value.bind(py).as_any())?)
     }
 
-    fn __hash__(&self, py: Python<'_>) -> PyResult<isize> {
-        // Hash on the same fields `__eq__` compares, and let Python
-        // hash the text so the two agree for every string it can hold.
-        let mut h = self.value.bind(py).as_any().hash()?;
-        h = h.wrapping_mul(1_000_003).wrapping_add(self.offset as isize);
-        h = h.wrapping_mul(1_000_003).wrapping_add(self.length as isize);
-        Ok(h)
+    /// Deliberately absent: defining `__eq__` without `__hash__`
+    /// makes a class unhashable in Python, and the pure-Python
+    /// `LiteralNode` does exactly that.  `Node` is unhashable on both
+    /// backends too, so no parse-tree node is hashable anywhere.
+    /// Adding one here made `set(literal_nodes)` work under the Rust
+    /// backend and raise `TypeError` under the other.
+    #[classattr]
+    #[pyo3(name = "__hash__")]
+    fn __hash__() -> Option<Py<PyAny>> {
+        None
     }
 }
 
@@ -122,7 +125,7 @@ impl PyLiteralNode {
 // Node
 // ----------------------------------------------------------------
 
-#[pyclass(name = "Node", module = "abnf_rust._ext")]
+#[pyclass(name = "Node", module = "abnf_rust._ext", subclass)]
 #[derive(Debug)]
 pub struct PyNode {
     #[pyo3(get)]
@@ -167,8 +170,24 @@ impl PyNode {
     }
 
     fn __richcmp__(&self, py: Python<'_>, other: &Self, op: CompareOp) -> PyResult<bool> {
+        // Structural, as the pure-Python `Node.__eq__` is: name and
+        // children, compared recursively.  Comparing concatenated
+        // values instead called two different parse trees equal
+        // whenever they spanned the same text -- `"xy" / ("x" "y")`
+        // against `("x" "y") / "xy"`, say -- which silently changed
+        // what a user's assertions meant depending on the backend.
         let eq = self.name == other.name
-            && self.value.bind(py).as_any().eq(other.value.bind(py).as_any())?;
+            && self.children.len() == other.children.len()
+            && {
+                let mut equal = true;
+                for (a, b) in self.children.iter().zip(other.children.iter()) {
+                    if !a.bind(py).eq(b.bind(py))? {
+                        equal = false;
+                        break;
+                    }
+                }
+                equal
+            };
         Ok(match op {
             CompareOp::Eq => eq,
             CompareOp::Ne => !eq,
@@ -187,18 +206,55 @@ impl PyNode {
     /// rather than a concatenation of the children's values: a node
     /// covers a contiguous run of the source, so the two are the same
     /// text.
-    pub fn from_rust(py: Python<'_>, n: &Node, source: &Bound<'_, PyString>) -> PyResult<Self> {
+    pub fn from_rust(
+        py: Python<'_>,
+        n: &Node,
+        source: &Bound<'_, PyString>,
+    ) -> PyResult<(Self, bool)> {
         let mut children: Vec<Py<PyAny>> = Vec::with_capacity(n.children.len());
+        let mut foreign = false;
         for child in n.children.iter() {
-            children.push(node_kind_to_py(py, child, source)?);
+            let (obj, child_foreign) = node_kind_to_py(py, child, source)?;
+            foreign |= child_foreign;
+            children.push(obj);
         }
-        let value = span_value(py, n.children.iter(), source)?;
-        Ok(Self {
-            name: n.name.as_ref().to_string(),
-            value: value.unbind(),
-            children,
-        })
+        // One slice of the source covers this node -- unless something
+        // below it carries its own text, in which case join the parts,
+        // which the children have already computed (issue #220).
+        let value = if foreign {
+            join_child_values(py, &children)?
+        } else {
+            span_value(py, n.children.iter(), source)?
+        };
+        Ok((
+            Self {
+                name: n.name.as_ref().to_string(),
+                value: value.unbind(),
+                children,
+            },
+            foreign,
+        ))
+    }
+}
+
+/// Concatenate the `value` of each already-converted child.  Used only
+/// for a node with a foreign descendant, where slicing the source
+/// would not reproduce the text.
+fn join_child_values<'py>(
+    py: Python<'py>,
+    children: &[Py<PyAny>],
+) -> PyResult<Bound<'py, PyString>> {
+    let mut joined = PyString::new(py, "").unbind();
+    for child in children {
+        let part = child.bind(py).getattr("value")?;
+        joined = joined
+            .bind(py)
+            .as_any()
+            .add(part)?
+            .cast_into::<PyString>()?
+            .unbind();
     }
+    Ok(joined.into_bound(py))
 }
 
 /// The text spanned by `nodes`: from the first descendant literal to
@@ -225,15 +281,44 @@ where
     }
 }
 
-/// Convert a `NodeKind` to a Python object.
+/// Convert a `NodeKind` to a Python object, reporting whether the
+/// subtree contains a node whose text is not a span of the source.
+///
+/// The flag is accumulated on the way up rather than recomputed by
+/// walking each subtree, which would make building a tree quadratic --
+/// measured at 13-17% on the parse benchmarks when this fix was first
+/// written that way.
 fn node_kind_to_py(
     py: Python<'_>,
     kind: &NodeKind,
     source: &Bound<'_, PyString>,
-) -> PyResult<Py<PyAny>> {
+) -> PyResult<(Py<PyAny>, bool)> {
     Ok(match kind {
-        NodeKind::Internal(n) => Py::new(py, PyNode::from_rust(py, n, source)?)?.into_any(),
-        NodeKind::Literal(l) => Py::new(py, PyLiteralNode::from_rust(l, source)?)?.into_any(),
+        NodeKind::Internal(n) => {
+            let (node, foreign) = PyNode::from_rust(py, n, source)?;
+            (Py::new(py, node)?.into_any(), foreign)
+        }
+        NodeKind::Literal(l) => (
+            Py::new(py, PyLiteralNode::from_rust(l, source)?)?.into_any(),
+            false,
+        ),
+        // Carries its own text, so it is handed back verbatim rather
+        // than re-sliced from the source (issue #220).
+        NodeKind::Foreign(f) => {
+            let value = from_code_points(py, &f.value)?;
+            (
+                Py::new(
+                    py,
+                    PyLiteralNode {
+                        value: value.unbind(),
+                        offset: f.offset,
+                        length: f.length,
+                    },
+                )?
+                .into_any(),
+                true,
+            )
+        }
     })
 }
 
@@ -242,7 +327,7 @@ fn node_kind_to_py(
 // Match
 // ----------------------------------------------------------------
 
-#[pyclass(name = "Match", module = "abnf_rust._ext")]
+#[pyclass(name = "Match", module = "abnf_rust._ext", subclass)]
 #[derive(Debug)]
 pub struct PyMatch {
     /// Match nodes as Python objects.
@@ -301,10 +386,17 @@ impl PyMatch {
 impl PyMatch {
     pub fn from_rust(py: Python<'_>, m: &Match, source: &Bound<'_, PyString>) -> PyResult<Self> {
         let mut nodes = Vec::with_capacity(m.nodes.len());
+        let mut foreign = false;
         for nk in &m.nodes {
-            nodes.push(node_kind_to_py(py, nk, source)?);
+            let (obj, node_foreign) = node_kind_to_py(py, nk, source)?;
+            foreign |= node_foreign;
+            nodes.push(obj);
         }
-        let cached_value = span_value(py, m.nodes.iter(), source)?;
+        let cached_value = if foreign {
+            join_child_values(py, &nodes)?
+        } else {
+            span_value(py, m.nodes.iter(), source)?
+        };
         Ok(Self {
             nodes,
             start: m.start,
@@ -339,11 +431,23 @@ fn py_to_node_kind(obj: &Bound<'_, PyAny>) -> PyResult<NodeKind> {
     //
     // A terminal contributes only its span; the text itself is read
     // back out of the source when the tree is rebuilt for Python.
+    // Terminals coming *in* from Python keep their own text.  The
+    // engine's own terminals are spans of the source, which is what
+    // lets their values be produced by slicing -- but a custom parser
+    // may return a normalised or synthesised value, and discarding it
+    // silently replaced it with unrelated source text (issue #220).
     if let Ok(lit) = obj.cast::<PyLiteralNode>() {
         let lit = lit.borrow();
-        return Ok(NodeKind::Literal(LiteralNode::new(lit.offset, lit.length)));
+        let value = CodePoints::new(lit.value.bind(obj.py()))?;
+        return Ok(NodeKind::Foreign(Arc::new(ForeignNode::new(
+            value.as_slice().into(),
+            lit.offset,
+            lit.length,
+        ))));
     }
-    if let (Ok(offset_obj), Ok(length_obj)) = (obj.getattr("offset"), obj.getattr("length")) {
+    if let (Ok(value_obj), Ok(offset_obj), Ok(length_obj)) =
+        (obj.getattr("value"), obj.getattr("offset"), obj.getattr("length"))
+    {
         if let (Ok(offset), Ok(length)) = (
             offset_obj.extract::<usize>(),
             length_obj.extract::<usize>(),
@@ -353,7 +457,13 @@ fn py_to_node_kind(obj: &Bound<'_, PyAny>) -> PyResult<NodeKind> {
                 .and_then(|n| n.extract())
                 .unwrap_or_else(|_| "literal".to_string());
             if name == "literal" {
-                return Ok(NodeKind::Literal(LiteralNode::new(offset, length)));
+                let value_str = value_obj.cast::<PyString>()?;
+                let value = CodePoints::new(value_str)?;
+                return Ok(NodeKind::Foreign(Arc::new(ForeignNode::new(
+                    value.as_slice().into(),
+                    offset,
+                    length,
+                ))));
             }
         }
     }

{abnf_rust-2.8.2 → abnf_rust-2.9.0}/rust/pyo3/src/parsers.rs

@@ -8,7 +8,8 @@ use std::sync::Arc;
 
 use pyo3::exceptions::PyTypeError;
 use pyo3::prelude::*;
-use pyo3::types::{PyString, PyTuple};
+use pyo3::sync::PyOnceLock;
+use pyo3::types::{PyFloat, PyInt, PyString, PyTuple, PyType};
 
 use abnf_core::{
     arc, Alternation, ArcParser, Concatenation, Literal, LiteralKind, OptionParser, Parser, Prose,
@@ -37,8 +38,78 @@ pub struct PyRepeat {
 #[pymethods]
 impl PyRepeat {
     #[new]
-    #[pyo3(signature = (min=0, max=None))]
-    fn new(py: Python<'_>, min: usize, max: Option<usize>) -> PyResult<Self> {
+    #[pyo3(signature = (min=None, max=None))]
+    fn new(
+        py: Python<'_>,
+        min: Option<&Bound<'_, PyAny>>,
+        max: Option<&Bound<'_, PyAny>>,
+    ) -> PyResult<Self> {
+        // A negative `min` is not an error.  The pure-Python `Repeat`
+        // accepts one, and `Repetition` then builds no mandatory
+        // prefix at all -- so it behaves as zero, which is what this
+        // clamps to (issue #221).  Rejecting it made the backends
+        // disagree over a value neither treats as meaningful.
+        let min: usize = match min {
+            None => 0,
+            Some(obj) => match obj.extract::<usize>() {
+                Ok(value) => value,
+                Err(err) => {
+                    if obj.cast::<PyInt>().is_ok() && obj.lt(0i64)? {
+                        0
+                    } else {
+                        return Err(err);
+                    }
+                }
+            },
+        };
+        // A bound too large for `usize` saturates rather than raising.
+        // Python's ints are unbounded, so `2*99999999999999999999"x"`
+        // is odd but valid ABNF that the pure-Python backend parses
+        // happily -- the bound is simply never reached.  Raising
+        // `OverflowError` here made the two backends disagree, and
+        // `OverflowError` is neither `GrammarError` nor `ParseError`,
+        // so it escaped the documented exception contract as well.
+        // No input can reach `usize::MAX` repetitions, so saturating
+        // is indistinguishable from the unbounded value.
+        let max: Option<usize> = match max {
+            None => None,
+            Some(obj) if obj.is_none() => None,
+            Some(obj) => match obj.extract::<usize>() {
+                Ok(value) => Some(value),
+                // A float bound is not an error either.  Pure Python
+                // compares `max` against the repetition count, so a
+                // non-integral float never equals it and the
+                // repetition is unbounded; an integral one caps where
+                // the integer would.  Mirror both.
+                // Test for a *float* specifically: an int converts to
+                // f64 too, and routing negative ints through here would
+                // swallow the `max < min` check below.
+                Err(_) if obj.is_instance_of::<PyFloat>() => {
+                    let value = obj.extract::<f64>()?;
+                    if value.is_finite() && value.fract() == 0.0 && value >= 0.0 {
+                        Some(value as usize)
+                    } else {
+                        None
+                    }
+                }
+                // Not an integer at all: report that, not a bound problem.
+                Err(err) if obj.cast::<PyInt>().is_err() => return Err(err),
+                // An integer outside `usize`.  Negative falls through to
+                // the `max < min` check below, exactly as in Python;
+                // anything else is bigger than any input can reach.
+                Err(_) if obj.lt(0i64)? => {
+                    let msg = format!(
+                        "Repeat max ({}) is less than min ({min}); \
+                         a repetition of {min}*{} can never match.",
+                        obj.str()?,
+                        obj.str()?
+                    );
+                    return Err(crate::errors::grammar_error(py, &msg));
+                }
+                Err(_) => Some(usize::MAX),
+            },
+        };
+
         // Mirrors the check in the pure-Python `Repeat.__init__`:
         // `3*2` is an impossible range, and leaving it unvalidated
         // silently behaves as `3*`.  Both backends must reject it
@@ -97,7 +168,9 @@ impl PyAlternation {
         start: usize,
     ) -> PyResult<Py<LparseIter>> {
         let cps = CodePoints::new(source)?;
-        let result = crate::recursion::call_lparse(|| self.inner.lparse(cps.as_slice(), start))?;
+        let result = crate::recursion::call_lparse(source.as_ptr() as usize, || {
+            self.inner.lparse(cps.as_slice(), start)
+        })?;
         lparse_iter(py, result, source)
     }
 
@@ -150,7 +223,9 @@ impl PyConcatenation {
         start: usize,
     ) -> PyResult<Py<LparseIter>> {
         let cps = CodePoints::new(source)?;
-        let result = crate::recursion::call_lparse(|| self.inner.lparse(cps.as_slice(), start))?;
+        let result = crate::recursion::call_lparse(source.as_ptr() as usize, || {
+            self.inner.lparse(cps.as_slice(), start)
+        })?;
         lparse_iter(py, result, source)
     }
 
@@ -186,7 +261,9 @@ impl PyRepetition {
         start: usize,
     ) -> PyResult<Py<LparseIter>> {
         let cps = CodePoints::new(source)?;
-        let result = crate::recursion::call_lparse(|| self.inner.lparse(cps.as_slice(), start))?;
+        let result = crate::recursion::call_lparse(source.as_ptr() as usize, || {
+            self.inner.lparse(cps.as_slice(), start)
+        })?;
         lparse_iter(py, result, source)
     }
 
@@ -222,7 +299,9 @@ impl PyOption {
         start: usize,
     ) -> PyResult<Py<LparseIter>> {
         let cps = CodePoints::new(source)?;
-        let result = crate::recursion::call_lparse(|| self.inner.lparse(cps.as_slice(), start))?;
+        let result = crate::recursion::call_lparse(source.as_ptr() as usize, || {
+            self.inner.lparse(cps.as_slice(), start)
+        })?;
         lparse_iter(py, result, source)
     }
 
@@ -269,7 +348,11 @@ impl PyLiteral {
                 };
                 return Ok(Self {
                     inner: Literal::range(*lo, *hi).into(),
-                    case_sensitive: true,
+                    // A range compares by code point either way, so
+                    // this attribute is inert -- but the pure-Python
+                    // constructor leaves it at its default, and the
+                    // two backends should read alike.
+                    case_sensitive,
                 });
             }
         }
@@ -285,7 +368,9 @@ impl PyLiteral {
         start: usize,
     ) -> PyResult<Py<LparseIter>> {
         let cps = CodePoints::new(source)?;
-        let result = crate::recursion::call_lparse(|| self.inner.lparse(cps.as_slice(), start))?;
+        let result = crate::recursion::call_lparse(source.as_ptr() as usize, || {
+            self.inner.lparse(cps.as_slice(), start)
+        })?;
         lparse_iter(py, result, source)
     }
 
@@ -347,7 +432,9 @@ impl PyProse {
         start: usize,
     ) -> PyResult<Py<LparseIter>> {
         let cps = CodePoints::new(source)?;
-        match crate::recursion::call_lparse(|| self.inner.lparse(cps.as_slice(), start))? {
+        match crate::recursion::call_lparse(source.as_ptr() as usize, || {
+            self.inner.lparse(cps.as_slice(), start)
+        })? {
             Ok(_) => lparse_iter(py, Ok(smallvec::SmallVec::new()), source),
             Err(err) => Err(parse_error_to_pyerr(py, err)),
         }
@@ -386,19 +473,31 @@ pub fn extract_parser(obj: &Bound<'_, PyAny>) -> PyResult<ArcParser> {
     if let Ok(p) = obj.cast::<PyProse>() {
         return Ok(p.borrow().inner.clone());
     }
-    // If the value is a Python Rule (or anything else carrying a
-    // `name`+`lparse` shape that fits the parser-by-name contract),
-    // look up — or lazily create — its shadow Rust `NamedRule` in
-    // the bridge registry.  This is the fast path that keeps rule
-    // references purely in Rust at parse time, instead of dispatching
-    // every reference through Python.
-    if obj.hasattr("name")? && obj.hasattr("lparse")? {
+    // If the value is a Python `Rule`, look up — or lazily create —
+    // its shadow Rust `NamedRule` in the bridge registry.  This is the
+    // fast path that keeps rule references purely in Rust at parse
+    // time, instead of dispatching every reference through Python.
+    //
+    // The test is `isinstance`, not the presence of `name` + `lparse`.
+    // Attribute shape caught anything with a `name`, which was wrong
+    // twice over (issue #203): such an object became a
+    // definition-less `NamedRule` whose own `lparse` was then never
+    // called, and it was entered into a registry keyed by the object's
+    // *address* — sound only because `Rule._obj_map` makes rules
+    // immortal, which is not true of an arbitrary user object.  A
+    // freed one's address could be handed to a new `Rule`, which then
+    // inherited its stale handle; defining that rule wrote into a
+    // parser tree built from an unrelated object.
+    static RULE_TYPE: PyOnceLock<Py<PyType>> = PyOnceLock::new();
+    let rule_type = RULE_TYPE.import(obj.py(), "abnf._parser_python", "Rule")?;
+    if obj.is_instance(rule_type)? {
         let handle = crate::bridge::get_or_create(obj)?;
         return Ok(arc(handle));
     }
-    // Last-resort fallback: any other object with an `lparse` method
-    // is wrapped as a `PyCallbackParser`.  Reserved for callers that
-    // pass duck-typed parsers without a `name` attribute.
+    // Any other object with an `lparse` method is wrapped as a
+    // `PyCallbackParser`, which holds a reference to it -- so unlike a
+    // bridge entry, it cannot outlive the object it refers to.  This
+    // is where every duck-typed parser goes, with or without a `name`.
     if obj.hasattr("lparse")? {
         let callback = PyCallbackParser::new(obj.clone().unbind(), "PyCallback");
         let parser: Parser = Parser::External(Arc::new(callback));

{abnf_rust-2.8.2 → abnf_rust-2.9.0}/rust/pyo3/src/recursion.rs

@@ -24,7 +24,7 @@
 //! tags into the right Python exception.  Other panics resume so
 //! they keep surfacing as `PanicException` via PyO3's defaults.
 
-use std::cell::RefCell;
+use std::cell::{Cell, RefCell};
 use std::panic::{self, AssertUnwindSafe};
 use std::sync::Once;
 
@@ -48,6 +48,17 @@ const PYERR_PANIC_TAG: &str = "pycallback-python-error";
 const UNDEFINED_RULE_TAG: &str = "Undefined rule";
 
 thread_local! {
+    /// Identity of the source the innermost active parse is running
+    /// over: the address of the Python `str` object, which is what
+    /// the pure-Python backend compares (`ctx[0] is source`) before
+    /// trusting a memo entry.  Zero when no parse is active.
+    ///
+    /// Not the code-point buffer's address: a nested call widens the
+    /// same `str` into a different pooled buffer, so that would report
+    /// "different source" for every re-entrant call and throw away the
+    /// enclosing parse's memo each time.
+    static CURRENT_SOURCE_ID: Cell<usize> = const { Cell::new(0) };
+
     /// Holds the most recent `PyErr` raised by a `PyCallbackParser`
     /// callback that should propagate (i.e. not a `ParseError`).
     /// Taken by `call_lparse` when it recognises the
@@ -57,6 +68,16 @@ thread_local! {
 
 static HOOK_INSTALLED: Once = Once::new();
 
+/// Restores the enclosing parse's source identity on the way out, so
+/// a nested call cannot leave the wrong one behind.
+struct RestoreSourceId(usize);
+
+impl Drop for RestoreSourceId {
+    fn drop(&mut self) {
+        CURRENT_SOURCE_ID.with(|id| id.set(self.0));
+    }
+}
+
 /// Install the panic hook exactly once per process.
 pub fn install_panic_hook() {
     HOOK_INSTALLED.call_once(|| {
@@ -99,7 +120,7 @@ pub fn propagate_pyerr(err: PyErr) -> ! {
 /// PyCallbackParser-stash panic to the saved `PyErr`.  Any other
 /// panic is re-raised so it surfaces normally via PyO3's
 /// `PanicException`.
-pub fn call_lparse<F>(f: F) -> PyResult<abnf_core::ParseResult>
+pub fn call_lparse<F>(source_id: usize, f: F) -> PyResult<abnf_core::ParseResult>
 where
     F: FnOnce() -> abnf_core::ParseResult,
 {
@@ -112,6 +133,24 @@ where
     // identity.  Nested entries -- a `PyCallbackParser` calling back
     // in -- share the epoch, so intra-parse reuse is preserved.
     let _scope = abnf_core::ParseScope::enter();
+
+    // ...but only while they are parsing the same text.  A custom
+    // parser may call back in on any source it likes, and cache
+    // entries are keyed by position alone, so sharing an epoch across
+    // two sources let the inner parse answer the outer one's lookups
+    // and the outer parse returned a wrong result (issue #202).
+    //
+    // Compare the `str` object's identity, exactly as the pure-Python
+    // memo does, and give a genuinely different source its own epoch.
+    // Same-source re-entry -- the common case -- keeps sharing, which
+    // matters because an epoch change resets the caches it touches:
+    // claiming one unconditionally would wipe the enclosing parse's
+    // memo on every callback.
+    let previous_source = CURRENT_SOURCE_ID.with(Cell::get);
+    let _source_scope = (previous_source != source_id)
+        .then(abnf_core::SourceScope::enter);
+    CURRENT_SOURCE_ID.with(|id| id.set(source_id));
+    let _restore = RestoreSourceId(previous_source);
     match panic::catch_unwind(AssertUnwindSafe(f)) {
         Ok(result) => Ok(result),
         Err(payload) => {