abnf-rust 2.8.2__tar.gz → 2.8.3__tar.gz

{abnf_rust-2.8.2 → abnf_rust-2.8.3}/Cargo.lock

@@ -4,7 +4,7 @@ version = 3
 
 [[package]]
 name = "abnf-core"
-version = "2.8.2"
+version = "2.8.3"
 dependencies = [
  "caseless",
  "lru",
@@ -14,7 +14,7 @@ dependencies = [
 
 [[package]]
 name = "abnf-rust-ext"
-version = "2.8.2"
+version = "2.8.3"
 dependencies = [
  "abnf-core",
  "mimalloc",
@@ -100,9 +100,9 @@ dependencies = [
 
 [[package]]
 name = "lru"
-version = "0.18.1"
+version = "0.18.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0b6180140927ee907000b0aa540091f6ea512ead4447c92b8fc35bc72788a5a6"
+checksum = "5d2f2f9b4ba7e6b24d95e7e899329d35be83bcded72c8540cdd5368932d1d90a"
 dependencies = [
  "hashbrown",
 ]

{abnf_rust-2.8.2 → abnf_rust-2.8.3}/Cargo.toml

@@ -6,7 +6,7 @@ resolver = "2"
 # Lock-stepped with the `abnf` distribution; both packages release
 # under the same git tag so the wheels at `pip install abnf==X` and
 # `pip install abnf-rust==X` always come from the same source state.
-version = "2.8.2"
+version = "2.8.3"
 edition = "2021"
 rust-version = "1.75"
 license = "MIT"
@@ -14,7 +14,7 @@ authors = ["Charles Yeomans <charles@declaresub.com>"]
 repository = "https://github.com/declaresub/abnf"
 
 [workspace.dependencies]
-abnf-core = { path = "rust/core", version = "2.8.2" }
+abnf-core = { path = "rust/core", version = "2.8.3" }
 caseless = "0.2"
 lru = "0.18"
 mimalloc = { version = "0.1", default-features = false }

{abnf_rust-2.8.2 → abnf_rust-2.8.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: abnf-rust
-Version: 2.8.2
+Version: 2.8.3
 Classifier: Development Status :: 4 - Beta
 Classifier: Intended Audience :: Developers
 Classifier: License :: OSI Approved :: MIT License

{abnf_rust-2.8.2 → abnf_rust-2.8.3}/rust/core/src/rule.rs

@@ -268,7 +268,17 @@ impl NamedRule {
                 self.name
             ),
         });
-        let def = self.definition().ok_or_else(|| self.parse_error(start))?;
+        // A rule that was never defined is a broken grammar, not input
+        // that failed to match.  Returning a `ParseError` here would be
+        // indistinguishable from "this alternative did not match", so
+        // an enclosing `Alternation` or `Repetition` would swallow it
+        // as backtracking and a typo'd rule name would silently delete
+        // that branch of the grammar.  The pure-Python backend raises
+        // `GrammarError`; panic with the phrase the PyO3 layer maps to
+        // it, the same route `is_excluded` and the recursion guard use.
+        let Some(def) = self.definition() else {
+            panic!("Undefined rule \"{}\"", self.name);
+        };
         let inner = def.lparse(source, start)?;
 
         let excluded = self.exclude();
@@ -296,9 +306,26 @@ impl NamedRule {
             if let Some(ref excluded) = excluded {
                 // `m.start` is the end of the match; the span is what
                 // the Python side rebuilds by joining node values.
-                if Self::is_excluded(excluded, &source[start..m.start]) {
-                    seen.remove(&m.start);
-                    continue;
+                //
+                // It is not necessarily a real span.  When this rule's
+                // definition is a `PyCallbackParser`, the offset comes
+                // from a Python parser and can be anything -- past the
+                // end of the source, or before `start`.  Slicing on it
+                // panicked, and a panic crosses the FFI as
+                // `PanicException`, a `BaseException` that `except
+                // ParseError` (or even `except Exception`) does not
+                // catch (issue #218).
+                //
+                // Text that is not in the source cannot be text the
+                // excluded rule matches, so a nonsensical span means
+                // "not excluded".  The bad offset then fails naturally
+                // further up, which is what the pure-Python backend
+                // does with the same input.
+                if let Some(span) = source.get(start..m.start) {
+                    if Self::is_excluded(excluded, span) {
+                        seen.remove(&m.start);
+                        continue;
+                    }
                 }
             }
             let node = Node::new(self.name.clone(), m.nodes.into_vec());

{abnf_rust-2.8.2 → abnf_rust-2.8.3}/rust/pyo3/src/nodes.rs

@@ -94,13 +94,16 @@ impl PyLiteralNode {
             && self.value.bind(py).as_any().eq(other.value.bind(py).as_any())?)
     }
 
-    fn __hash__(&self, py: Python<'_>) -> PyResult<isize> {
-        // Hash on the same fields `__eq__` compares, and let Python
-        // hash the text so the two agree for every string it can hold.
-        let mut h = self.value.bind(py).as_any().hash()?;
-        h = h.wrapping_mul(1_000_003).wrapping_add(self.offset as isize);
-        h = h.wrapping_mul(1_000_003).wrapping_add(self.length as isize);
-        Ok(h)
+    /// Deliberately absent: defining `__eq__` without `__hash__`
+    /// makes a class unhashable in Python, and the pure-Python
+    /// `LiteralNode` does exactly that.  `Node` is unhashable on both
+    /// backends too, so no parse-tree node is hashable anywhere.
+    /// Adding one here made `set(literal_nodes)` work under the Rust
+    /// backend and raise `TypeError` under the other.
+    #[classattr]
+    #[pyo3(name = "__hash__")]
+    fn __hash__() -> Option<Py<PyAny>> {
+        None
     }
 }
 
@@ -167,8 +170,24 @@ impl PyNode {
     }
 
     fn __richcmp__(&self, py: Python<'_>, other: &Self, op: CompareOp) -> PyResult<bool> {
+        // Structural, as the pure-Python `Node.__eq__` is: name and
+        // children, compared recursively.  Comparing concatenated
+        // values instead called two different parse trees equal
+        // whenever they spanned the same text -- `"xy" / ("x" "y")`
+        // against `("x" "y") / "xy"`, say -- which silently changed
+        // what a user's assertions meant depending on the backend.
         let eq = self.name == other.name
-            && self.value.bind(py).as_any().eq(other.value.bind(py).as_any())?;
+            && self.children.len() == other.children.len()
+            && {
+                let mut equal = true;
+                for (a, b) in self.children.iter().zip(other.children.iter()) {
+                    if !a.bind(py).eq(b.bind(py))? {
+                        equal = false;
+                        break;
+                    }
+                }
+                equal
+            };
         Ok(match op {
             CompareOp::Eq => eq,
             CompareOp::Ne => !eq,

{abnf_rust-2.8.2 → abnf_rust-2.8.3}/rust/pyo3/src/parsers.rs

@@ -8,7 +8,8 @@ use std::sync::Arc;
 
 use pyo3::exceptions::PyTypeError;
 use pyo3::prelude::*;
-use pyo3::types::{PyString, PyTuple};
+use pyo3::sync::PyOnceLock;
+use pyo3::types::{PyInt, PyString, PyTuple, PyType};
 
 use abnf_core::{
     arc, Alternation, ArcParser, Concatenation, Literal, LiteralKind, OptionParser, Parser, Prose,
@@ -38,7 +39,39 @@ pub struct PyRepeat {
 impl PyRepeat {
     #[new]
     #[pyo3(signature = (min=0, max=None))]
-    fn new(py: Python<'_>, min: usize, max: Option<usize>) -> PyResult<Self> {
+    fn new(py: Python<'_>, min: usize, max: Option<&Bound<'_, PyAny>>) -> PyResult<Self> {
+        // A bound too large for `usize` saturates rather than raising.
+        // Python's ints are unbounded, so `2*99999999999999999999"x"`
+        // is odd but valid ABNF that the pure-Python backend parses
+        // happily -- the bound is simply never reached.  Raising
+        // `OverflowError` here made the two backends disagree, and
+        // `OverflowError` is neither `GrammarError` nor `ParseError`,
+        // so it escaped the documented exception contract as well.
+        // No input can reach `usize::MAX` repetitions, so saturating
+        // is indistinguishable from the unbounded value.
+        let max: Option<usize> = match max {
+            None => None,
+            Some(obj) if obj.is_none() => None,
+            Some(obj) => match obj.extract::<usize>() {
+                Ok(value) => Some(value),
+                // Not an integer at all: report that, not a bound problem.
+                Err(err) if obj.cast::<PyInt>().is_err() => return Err(err),
+                // An integer outside `usize`.  Negative falls through to
+                // the `max < min` check below, exactly as in Python;
+                // anything else is bigger than any input can reach.
+                Err(_) if obj.lt(0i64)? => {
+                    let msg = format!(
+                        "Repeat max ({}) is less than min ({min}); \
+                         a repetition of {min}*{} can never match.",
+                        obj.str()?,
+                        obj.str()?
+                    );
+                    return Err(crate::errors::grammar_error(py, &msg));
+                }
+                Err(_) => Some(usize::MAX),
+            },
+        };
+
         // Mirrors the check in the pure-Python `Repeat.__init__`:
         // `3*2` is an impossible range, and leaving it unvalidated
         // silently behaves as `3*`.  Both backends must reject it
@@ -97,7 +130,9 @@ impl PyAlternation {
         start: usize,
     ) -> PyResult<Py<LparseIter>> {
         let cps = CodePoints::new(source)?;
-        let result = crate::recursion::call_lparse(|| self.inner.lparse(cps.as_slice(), start))?;
+        let result = crate::recursion::call_lparse(source.as_ptr() as usize, || {
+            self.inner.lparse(cps.as_slice(), start)
+        })?;
         lparse_iter(py, result, source)
     }
 
@@ -150,7 +185,9 @@ impl PyConcatenation {
         start: usize,
     ) -> PyResult<Py<LparseIter>> {
         let cps = CodePoints::new(source)?;
-        let result = crate::recursion::call_lparse(|| self.inner.lparse(cps.as_slice(), start))?;
+        let result = crate::recursion::call_lparse(source.as_ptr() as usize, || {
+            self.inner.lparse(cps.as_slice(), start)
+        })?;
         lparse_iter(py, result, source)
     }
 
@@ -186,7 +223,9 @@ impl PyRepetition {
         start: usize,
     ) -> PyResult<Py<LparseIter>> {
         let cps = CodePoints::new(source)?;
-        let result = crate::recursion::call_lparse(|| self.inner.lparse(cps.as_slice(), start))?;
+        let result = crate::recursion::call_lparse(source.as_ptr() as usize, || {
+            self.inner.lparse(cps.as_slice(), start)
+        })?;
         lparse_iter(py, result, source)
     }
 
@@ -222,7 +261,9 @@ impl PyOption {
         start: usize,
     ) -> PyResult<Py<LparseIter>> {
         let cps = CodePoints::new(source)?;
-        let result = crate::recursion::call_lparse(|| self.inner.lparse(cps.as_slice(), start))?;
+        let result = crate::recursion::call_lparse(source.as_ptr() as usize, || {
+            self.inner.lparse(cps.as_slice(), start)
+        })?;
         lparse_iter(py, result, source)
     }
 
@@ -269,7 +310,11 @@ impl PyLiteral {
                 };
                 return Ok(Self {
                     inner: Literal::range(*lo, *hi).into(),
-                    case_sensitive: true,
+                    // A range compares by code point either way, so
+                    // this attribute is inert -- but the pure-Python
+                    // constructor leaves it at its default, and the
+                    // two backends should read alike.
+                    case_sensitive,
                 });
             }
         }
@@ -285,7 +330,9 @@ impl PyLiteral {
         start: usize,
     ) -> PyResult<Py<LparseIter>> {
         let cps = CodePoints::new(source)?;
-        let result = crate::recursion::call_lparse(|| self.inner.lparse(cps.as_slice(), start))?;
+        let result = crate::recursion::call_lparse(source.as_ptr() as usize, || {
+            self.inner.lparse(cps.as_slice(), start)
+        })?;
         lparse_iter(py, result, source)
     }
 
@@ -347,7 +394,9 @@ impl PyProse {
         start: usize,
     ) -> PyResult<Py<LparseIter>> {
         let cps = CodePoints::new(source)?;
-        match crate::recursion::call_lparse(|| self.inner.lparse(cps.as_slice(), start))? {
+        match crate::recursion::call_lparse(source.as_ptr() as usize, || {
+            self.inner.lparse(cps.as_slice(), start)
+        })? {
             Ok(_) => lparse_iter(py, Ok(smallvec::SmallVec::new()), source),
             Err(err) => Err(parse_error_to_pyerr(py, err)),
         }
@@ -386,19 +435,31 @@ pub fn extract_parser(obj: &Bound<'_, PyAny>) -> PyResult<ArcParser> {
     if let Ok(p) = obj.cast::<PyProse>() {
         return Ok(p.borrow().inner.clone());
     }
-    // If the value is a Python Rule (or anything else carrying a
-    // `name`+`lparse` shape that fits the parser-by-name contract),
-    // look up — or lazily create — its shadow Rust `NamedRule` in
-    // the bridge registry.  This is the fast path that keeps rule
-    // references purely in Rust at parse time, instead of dispatching
-    // every reference through Python.
-    if obj.hasattr("name")? && obj.hasattr("lparse")? {
+    // If the value is a Python `Rule`, look up — or lazily create —
+    // its shadow Rust `NamedRule` in the bridge registry.  This is the
+    // fast path that keeps rule references purely in Rust at parse
+    // time, instead of dispatching every reference through Python.
+    //
+    // The test is `isinstance`, not the presence of `name` + `lparse`.
+    // Attribute shape caught anything with a `name`, which was wrong
+    // twice over (issue #203): such an object became a
+    // definition-less `NamedRule` whose own `lparse` was then never
+    // called, and it was entered into a registry keyed by the object's
+    // *address* — sound only because `Rule._obj_map` makes rules
+    // immortal, which is not true of an arbitrary user object.  A
+    // freed one's address could be handed to a new `Rule`, which then
+    // inherited its stale handle; defining that rule wrote into a
+    // parser tree built from an unrelated object.
+    static RULE_TYPE: PyOnceLock<Py<PyType>> = PyOnceLock::new();
+    let rule_type = RULE_TYPE.import(obj.py(), "abnf._parser_python", "Rule")?;
+    if obj.is_instance(rule_type)? {
         let handle = crate::bridge::get_or_create(obj)?;
         return Ok(arc(handle));
     }
-    // Last-resort fallback: any other object with an `lparse` method
-    // is wrapped as a `PyCallbackParser`.  Reserved for callers that
-    // pass duck-typed parsers without a `name` attribute.
+    // Any other object with an `lparse` method is wrapped as a
+    // `PyCallbackParser`, which holds a reference to it -- so unlike a
+    // bridge entry, it cannot outlive the object it refers to.  This
+    // is where every duck-typed parser goes, with or without a `name`.
     if obj.hasattr("lparse")? {
         let callback = PyCallbackParser::new(obj.clone().unbind(), "PyCallback");
         let parser: Parser = Parser::External(Arc::new(callback));

{abnf_rust-2.8.2 → abnf_rust-2.8.3}/rust/pyo3/src/recursion.rs

@@ -24,7 +24,7 @@
 //! tags into the right Python exception.  Other panics resume so
 //! they keep surfacing as `PanicException` via PyO3's defaults.
 
-use std::cell::RefCell;
+use std::cell::{Cell, RefCell};
 use std::panic::{self, AssertUnwindSafe};
 use std::sync::Once;
 
@@ -48,6 +48,17 @@ const PYERR_PANIC_TAG: &str = "pycallback-python-error";
 const UNDEFINED_RULE_TAG: &str = "Undefined rule";
 
 thread_local! {
+    /// Identity of the source the innermost active parse is running
+    /// over: the address of the Python `str` object, which is what
+    /// the pure-Python backend compares (`ctx[0] is source`) before
+    /// trusting a memo entry.  Zero when no parse is active.
+    ///
+    /// Not the code-point buffer's address: a nested call widens the
+    /// same `str` into a different pooled buffer, so that would report
+    /// "different source" for every re-entrant call and throw away the
+    /// enclosing parse's memo each time.
+    static CURRENT_SOURCE_ID: Cell<usize> = const { Cell::new(0) };
+
     /// Holds the most recent `PyErr` raised by a `PyCallbackParser`
     /// callback that should propagate (i.e. not a `ParseError`).
     /// Taken by `call_lparse` when it recognises the
@@ -57,6 +68,16 @@ thread_local! {
 
 static HOOK_INSTALLED: Once = Once::new();
 
+/// Restores the enclosing parse's source identity on the way out, so
+/// a nested call cannot leave the wrong one behind.
+struct RestoreSourceId(usize);
+
+impl Drop for RestoreSourceId {
+    fn drop(&mut self) {
+        CURRENT_SOURCE_ID.with(|id| id.set(self.0));
+    }
+}
+
 /// Install the panic hook exactly once per process.
 pub fn install_panic_hook() {
     HOOK_INSTALLED.call_once(|| {
@@ -99,7 +120,7 @@ pub fn propagate_pyerr(err: PyErr) -> ! {
 /// PyCallbackParser-stash panic to the saved `PyErr`.  Any other
 /// panic is re-raised so it surfaces normally via PyO3's
 /// `PanicException`.
-pub fn call_lparse<F>(f: F) -> PyResult<abnf_core::ParseResult>
+pub fn call_lparse<F>(source_id: usize, f: F) -> PyResult<abnf_core::ParseResult>
 where
     F: FnOnce() -> abnf_core::ParseResult,
 {
@@ -112,6 +133,24 @@ where
     // identity.  Nested entries -- a `PyCallbackParser` calling back
     // in -- share the epoch, so intra-parse reuse is preserved.
     let _scope = abnf_core::ParseScope::enter();
+
+    // ...but only while they are parsing the same text.  A custom
+    // parser may call back in on any source it likes, and cache
+    // entries are keyed by position alone, so sharing an epoch across
+    // two sources let the inner parse answer the outer one's lookups
+    // and the outer parse returned a wrong result (issue #202).
+    //
+    // Compare the `str` object's identity, exactly as the pure-Python
+    // memo does, and give a genuinely different source its own epoch.
+    // Same-source re-entry -- the common case -- keeps sharing, which
+    // matters because an epoch change resets the caches it touches:
+    // claiming one unconditionally would wipe the enclosing parse's
+    // memo on every callback.
+    let previous_source = CURRENT_SOURCE_ID.with(Cell::get);
+    let _source_scope = (previous_source != source_id)
+        .then(abnf_core::SourceScope::enter);
+    CURRENT_SOURCE_ID.with(|id| id.set(source_id));
+    let _restore = RestoreSourceId(previous_source);
     match panic::catch_unwind(AssertUnwindSafe(f)) {
         Ok(result) => Ok(result),
         Err(payload) => {