lark-rust 0.2.0__tar.gz → 0.2.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (27) hide show
  1. {lark_rust-0.2.0 → lark_rust-0.2.1}/Cargo.lock +1 -1
  2. {lark_rust-0.2.0 → lark_rust-0.2.1}/Cargo.toml +1 -1
  3. {lark_rust-0.2.0 → lark_rust-0.2.1}/PKG-INFO +4 -4
  4. {lark_rust-0.2.0 → lark_rust-0.2.1}/README.md +3 -3
  5. {lark_rust-0.2.0 → lark_rust-0.2.1}/pyproject.toml +1 -1
  6. {lark_rust-0.2.0 → lark_rust-0.2.1}/python/lark_rust/__init__.py +14 -6
  7. {lark_rust-0.2.0 → lark_rust-0.2.1}/src/parser.rs +109 -1
  8. {lark_rust-0.2.0 → lark_rust-0.2.1}/.github/workflows/ci.yml +0 -0
  9. {lark_rust-0.2.0 → lark_rust-0.2.1}/.github/workflows/release.yml +0 -0
  10. {lark_rust-0.2.0 → lark_rust-0.2.1}/.gitignore +0 -0
  11. {lark_rust-0.2.0 → lark_rust-0.2.1}/LICENSE +0 -0
  12. {lark_rust-0.2.0 → lark_rust-0.2.1}/benchmarks/bench.py +0 -0
  13. {lark_rust-0.2.0 → lark_rust-0.2.1}/src/lexer.rs +0 -0
  14. {lark_rust-0.2.0 → lark_rust-0.2.1}/src/lexer_state.rs +0 -0
  15. {lark_rust-0.2.0 → lark_rust-0.2.1}/src/lexer_thread.rs +0 -0
  16. {lark_rust-0.2.0 → lark_rust-0.2.1}/src/lib.rs +0 -0
  17. {lark_rust-0.2.0 → lark_rust-0.2.1}/src/line_counter.rs +0 -0
  18. {lark_rust-0.2.0 → lark_rust-0.2.1}/src/scanner.rs +0 -0
  19. {lark_rust-0.2.0 → lark_rust-0.2.1}/src/token.rs +0 -0
  20. {lark_rust-0.2.0 → lark_rust-0.2.1}/tests/test_cython_compat.py +0 -0
  21. {lark_rust-0.2.0 → lark_rust-0.2.1}/tests/test_integration.py +0 -0
  22. {lark_rust-0.2.0 → lark_rust-0.2.1}/tests/test_lexer.py +0 -0
  23. {lark_rust-0.2.0 → lark_rust-0.2.1}/tests/test_lexer_state.py +0 -0
  24. {lark_rust-0.2.0 → lark_rust-0.2.1}/tests/test_lexer_thread.py +0 -0
  25. {lark_rust-0.2.0 → lark_rust-0.2.1}/tests/test_line_counter.py +0 -0
  26. {lark_rust-0.2.0 → lark_rust-0.2.1}/tests/test_scanner.py +0 -0
  27. {lark_rust-0.2.0 → lark_rust-0.2.1}/tests/test_token.py +0 -0
@@ -60,7 +60,7 @@ dependencies = [
60
60
 
61
61
  [[package]]
62
62
  name = "lark-rust"
63
- version = "0.2.0"
63
+ version = "0.2.1"
64
64
  dependencies = [
65
65
  "fancy-regex",
66
66
  "pyo3",
@@ -1,6 +1,6 @@
1
1
  [package]
2
2
  name = "lark-rust"
3
- version = "0.2.0"
3
+ version = "0.2.1"
4
4
  edition = "2021"
5
5
  readme = "README.md"
6
6
 
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: lark-rust
3
- Version: 0.2.0
3
+ Version: 0.2.1
4
4
  Classifier: Development Status :: 4 - Beta
5
5
  Classifier: Intended Audience :: Developers
6
6
  Classifier: License :: OSI Approved :: MIT License
@@ -56,9 +56,9 @@ Measured on Apple M4 Pro, Python 3.13, median of 20 iterations:
56
56
 
57
57
  | Workload | lark | lark-cython | lark-rust | vs lark | vs cython |
58
58
  |---|---|---|---|---|---|
59
- | JSON (3 KB nested) | 4.02 ms | 2.39 ms | **2.04 ms** | 2.0x | 1.2x |
60
- | Arithmetic (deep) | 3.22 ms | 1.98 ms | **1.29 ms** | 2.5x | 1.5x |
61
- | Lexer (10k words) | 28.66 ms | 16.50 ms | **11.44 ms** | 2.5x | 1.4x |
59
+ | JSON (3 KB nested) | 4.16 ms | 2.45 ms | **1.73 ms** | 2.4x | 1.4x |
60
+ | Arithmetic (deep) | 3.08 ms | 2.00 ms | **1.18 ms** | 2.6x | 1.7x |
61
+ | Lexer (10k words) | 28.41 ms | 16.89 ms | **10.61 ms** | 2.7x | 1.6x |
62
62
 
63
63
  Reproduce with `python benchmarks/bench.py` (install `lark-cython` for the three-way comparison).
64
64
 
@@ -26,9 +26,9 @@ Measured on Apple M4 Pro, Python 3.13, median of 20 iterations:
26
26
 
27
27
  | Workload | lark | lark-cython | lark-rust | vs lark | vs cython |
28
28
  |---|---|---|---|---|---|
29
- | JSON (3 KB nested) | 4.02 ms | 2.39 ms | **2.04 ms** | 2.0x | 1.2x |
30
- | Arithmetic (deep) | 3.22 ms | 1.98 ms | **1.29 ms** | 2.5x | 1.5x |
31
- | Lexer (10k words) | 28.66 ms | 16.50 ms | **11.44 ms** | 2.5x | 1.4x |
29
+ | JSON (3 KB nested) | 4.16 ms | 2.45 ms | **1.73 ms** | 2.4x | 1.4x |
30
+ | Arithmetic (deep) | 3.08 ms | 2.00 ms | **1.18 ms** | 2.6x | 1.7x |
31
+ | Lexer (10k words) | 28.41 ms | 16.89 ms | **10.61 ms** | 2.7x | 1.6x |
32
32
 
33
33
  Reproduce with `python benchmarks/bench.py` (install `lark-cython` for the three-way comparison).
34
34
 
@@ -4,7 +4,7 @@ build-backend = "maturin"
4
4
 
5
5
  [project]
6
6
  name = "lark-rust"
7
- version = "0.2.0"
7
+ version = "0.2.1"
8
8
  description = "High-performance Rust-accelerated drop-in replacement for lark's LALR parser stack"
9
9
  requires-python = ">=3.8"
10
10
  license = "MIT"
@@ -191,13 +191,23 @@ class _Parser:
191
191
 
192
192
  def parse_from_state(self, state, last_token=None):
193
193
  try:
194
- token = last_token
195
194
  lexer = state.lexer
195
+ compiled = state.parse_conf._compiled
196
+
197
+ if (last_token is None
198
+ and hasattr(lexer, 'state')
199
+ and isinstance(lexer.lexer, BasicLexer)):
200
+ # Fast path: LexerThread with BasicLexer — run entire
201
+ # lex+parse loop in a single Rust call.
202
+ return compiled.parse_loop(
203
+ lexer.lexer, lexer.state,
204
+ state.state_stack, state.value_stack,
205
+ state,
206
+ )
196
207
 
197
- # Inline the lexer loop: call next_token directly instead of
198
- # using the lex() generator. Avoids generator overhead.
208
+ # Fallback: ContextualLexer or resume with last_token
209
+ token = last_token
199
210
  if hasattr(lexer, 'state'):
200
- # LexerThread path
201
211
  inner_lexer = lexer.lexer
202
212
  lexer_state = lexer.state
203
213
  try:
@@ -207,8 +217,6 @@ class _Parser:
207
217
  except EOFError:
208
218
  pass
209
219
  else:
210
- # ContextualLexer path — uses lex() which needs
211
- # lexer_state + parser_state together
212
220
  for token in lexer.lex(state):
213
221
  assert token is not None
214
222
  state.feed_token(token)
@@ -2,6 +2,10 @@ use pyo3::prelude::*;
2
2
  use pyo3::types::{PyDict, PyList, PySet, PyTuple};
3
3
  use std::collections::HashMap;
4
4
 
5
+ use crate::lexer::BasicLexer;
6
+ use crate::lexer_state::LexerState;
7
+ use crate::token::Token;
8
+
5
9
  /// A pre-compiled rule: stores the expansion size and origin name
6
10
  /// so we never touch Python Rule objects during parsing.
7
11
  struct CompiledRule {
@@ -143,6 +147,7 @@ impl CompiledParseTable {
143
147
  /// Takes state_stack and value_stack as mutable Python lists,
144
148
  /// the token, and is_end flag.
145
149
  /// Returns None on shift, or the final value on successful end.
150
+ #[allow(clippy::too_many_arguments)]
146
151
  fn feed_token(
147
152
  &self,
148
153
  state_stack: &Bound<'_, PyList>,
@@ -152,6 +157,109 @@ impl CompiledParseTable {
152
157
  is_end: bool,
153
158
  parser_state: &Bound<'_, PyAny>,
154
159
  py: Python<'_>,
160
+ ) -> PyResult<Option<Py<PyAny>>> {
161
+ self.feed_token_inner(
162
+ state_stack,
163
+ value_stack,
164
+ token,
165
+ token_type,
166
+ is_end,
167
+ parser_state,
168
+ py,
169
+ )
170
+ }
171
+
172
+ /// Run the full lex-parse loop in Rust for the BasicLexer path.
173
+ /// Calls next_token and feed_token in a tight Rust loop, only
174
+ /// crossing back to Python for callbacks.
175
+ #[allow(clippy::too_many_arguments)]
176
+ fn parse_loop(
177
+ &self,
178
+ lexer_obj: &Bound<'_, PyAny>,
179
+ lexer_state_obj: &Bound<'_, PyAny>,
180
+ state_stack: &Bound<'_, PyList>,
181
+ value_stack: &Bound<'_, PyList>,
182
+ parser_state: &Bound<'_, PyAny>,
183
+ py: Python<'_>,
184
+ ) -> PyResult<Py<PyAny>> {
185
+ let mut last_token: Option<Token> = None;
186
+
187
+ // Tight lex-parse loop: next_token + feed_token without Python overhead
188
+ loop {
189
+ let result = {
190
+ let mut lexer = lexer_obj.downcast::<BasicLexer>()?.borrow_mut();
191
+ let mut lexer_state = lexer_state_obj.downcast::<LexerState>()?.borrow_mut();
192
+ lexer.next_token(&mut lexer_state, parser_state, py)
193
+ };
194
+ match result {
195
+ Ok(token) => {
196
+ let token_type = token.type_.clone();
197
+ let token_py = token.clone().into_pyobject(py)?.into_any();
198
+ last_token = Some(token);
199
+ self.feed_token_inner(
200
+ state_stack,
201
+ value_stack,
202
+ &token_py,
203
+ &token_type,
204
+ false,
205
+ parser_state,
206
+ py,
207
+ )?;
208
+ }
209
+ Err(e) if e.is_instance_of::<pyo3::exceptions::PyEOFError>(py) => {
210
+ break;
211
+ }
212
+ Err(e) => {
213
+ return Err(e);
214
+ }
215
+ }
216
+ }
217
+
218
+ // Feed $END token
219
+ let end_token = if let Some(ref lt) = last_token {
220
+ Token::new(
221
+ "$END".to_string(),
222
+ String::new(),
223
+ lt.start_pos,
224
+ lt.line,
225
+ lt.column,
226
+ lt.end_line,
227
+ lt.end_column,
228
+ lt.end_pos,
229
+ )
230
+ } else {
231
+ Token::new("$END".to_string(), String::new(), 0, 1, 1, None, None, None)
232
+ };
233
+ let end_token_py = end_token.into_pyobject(py)?.into_any();
234
+
235
+ let result = self.feed_token_inner(
236
+ state_stack,
237
+ value_stack,
238
+ &end_token_py,
239
+ "$END",
240
+ true,
241
+ parser_state,
242
+ py,
243
+ )?;
244
+
245
+ result.ok_or_else(|| {
246
+ pyo3::exceptions::PyRuntimeError::new_err("parse_loop: feed_token($END) returned None")
247
+ })
248
+ }
249
+ }
250
+
251
+ impl CompiledParseTable {
252
+ /// Shared feed_token implementation used by both feed_token and parse_loop.
253
+ #[allow(clippy::too_many_arguments)]
254
+ fn feed_token_inner(
255
+ &self,
256
+ state_stack: &Bound<'_, PyList>,
257
+ value_stack: &Bound<'_, PyList>,
258
+ token: &Bound<'_, PyAny>,
259
+ token_type: &str,
260
+ is_end: bool,
261
+ parser_state: &Bound<'_, PyAny>,
262
+ py: Python<'_>,
155
263
  ) -> PyResult<Option<Py<PyAny>>> {
156
264
  loop {
157
265
  let stack_len = state_stack.len();
@@ -209,7 +317,7 @@ impl CompiledParseTable {
209
317
  value_stack.del_slice(start, vstack_len)?;
210
318
  slice
211
319
  } else {
212
- PyList::empty(py).into()
320
+ PyList::empty(py)
213
321
  };
214
322
 
215
323
  let value = rule.callback.call1(py, (&children,))?;
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes