lark-rust 0.2.0__tar.gz → 0.2.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {lark_rust-0.2.0 → lark_rust-0.2.1}/Cargo.lock +1 -1
- {lark_rust-0.2.0 → lark_rust-0.2.1}/Cargo.toml +1 -1
- {lark_rust-0.2.0 → lark_rust-0.2.1}/PKG-INFO +4 -4
- {lark_rust-0.2.0 → lark_rust-0.2.1}/README.md +3 -3
- {lark_rust-0.2.0 → lark_rust-0.2.1}/pyproject.toml +1 -1
- {lark_rust-0.2.0 → lark_rust-0.2.1}/python/lark_rust/__init__.py +14 -6
- {lark_rust-0.2.0 → lark_rust-0.2.1}/src/parser.rs +109 -1
- {lark_rust-0.2.0 → lark_rust-0.2.1}/.github/workflows/ci.yml +0 -0
- {lark_rust-0.2.0 → lark_rust-0.2.1}/.github/workflows/release.yml +0 -0
- {lark_rust-0.2.0 → lark_rust-0.2.1}/.gitignore +0 -0
- {lark_rust-0.2.0 → lark_rust-0.2.1}/LICENSE +0 -0
- {lark_rust-0.2.0 → lark_rust-0.2.1}/benchmarks/bench.py +0 -0
- {lark_rust-0.2.0 → lark_rust-0.2.1}/src/lexer.rs +0 -0
- {lark_rust-0.2.0 → lark_rust-0.2.1}/src/lexer_state.rs +0 -0
- {lark_rust-0.2.0 → lark_rust-0.2.1}/src/lexer_thread.rs +0 -0
- {lark_rust-0.2.0 → lark_rust-0.2.1}/src/lib.rs +0 -0
- {lark_rust-0.2.0 → lark_rust-0.2.1}/src/line_counter.rs +0 -0
- {lark_rust-0.2.0 → lark_rust-0.2.1}/src/scanner.rs +0 -0
- {lark_rust-0.2.0 → lark_rust-0.2.1}/src/token.rs +0 -0
- {lark_rust-0.2.0 → lark_rust-0.2.1}/tests/test_cython_compat.py +0 -0
- {lark_rust-0.2.0 → lark_rust-0.2.1}/tests/test_integration.py +0 -0
- {lark_rust-0.2.0 → lark_rust-0.2.1}/tests/test_lexer.py +0 -0
- {lark_rust-0.2.0 → lark_rust-0.2.1}/tests/test_lexer_state.py +0 -0
- {lark_rust-0.2.0 → lark_rust-0.2.1}/tests/test_lexer_thread.py +0 -0
- {lark_rust-0.2.0 → lark_rust-0.2.1}/tests/test_line_counter.py +0 -0
- {lark_rust-0.2.0 → lark_rust-0.2.1}/tests/test_scanner.py +0 -0
- {lark_rust-0.2.0 → lark_rust-0.2.1}/tests/test_token.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: lark-rust
|
|
3
|
-
Version: 0.2.
|
|
3
|
+
Version: 0.2.1
|
|
4
4
|
Classifier: Development Status :: 4 - Beta
|
|
5
5
|
Classifier: Intended Audience :: Developers
|
|
6
6
|
Classifier: License :: OSI Approved :: MIT License
|
|
@@ -56,9 +56,9 @@ Measured on Apple M4 Pro, Python 3.13, median of 20 iterations:
|
|
|
56
56
|
|
|
57
57
|
| Workload | lark | lark-cython | lark-rust | vs lark | vs cython |
|
|
58
58
|
|---|---|---|---|---|---|
|
|
59
|
-
| JSON (3 KB nested) | 4.
|
|
60
|
-
| Arithmetic (deep) | 3.
|
|
61
|
-
| Lexer (10k words) | 28.
|
|
59
|
+
| JSON (3 KB nested) | 4.16 ms | 2.45 ms | **1.73 ms** | 2.4x | 1.4x |
|
|
60
|
+
| Arithmetic (deep) | 3.08 ms | 2.00 ms | **1.18 ms** | 2.6x | 1.7x |
|
|
61
|
+
| Lexer (10k words) | 28.41 ms | 16.89 ms | **10.61 ms** | 2.7x | 1.6x |
|
|
62
62
|
|
|
63
63
|
Reproduce with `python benchmarks/bench.py` (install `lark-cython` for the three-way comparison).
|
|
64
64
|
|
|
@@ -26,9 +26,9 @@ Measured on Apple M4 Pro, Python 3.13, median of 20 iterations:
|
|
|
26
26
|
|
|
27
27
|
| Workload | lark | lark-cython | lark-rust | vs lark | vs cython |
|
|
28
28
|
|---|---|---|---|---|---|
|
|
29
|
-
| JSON (3 KB nested) | 4.
|
|
30
|
-
| Arithmetic (deep) | 3.
|
|
31
|
-
| Lexer (10k words) | 28.
|
|
29
|
+
| JSON (3 KB nested) | 4.16 ms | 2.45 ms | **1.73 ms** | 2.4x | 1.4x |
|
|
30
|
+
| Arithmetic (deep) | 3.08 ms | 2.00 ms | **1.18 ms** | 2.6x | 1.7x |
|
|
31
|
+
| Lexer (10k words) | 28.41 ms | 16.89 ms | **10.61 ms** | 2.7x | 1.6x |
|
|
32
32
|
|
|
33
33
|
Reproduce with `python benchmarks/bench.py` (install `lark-cython` for the three-way comparison).
|
|
34
34
|
|
|
@@ -191,13 +191,23 @@ class _Parser:
|
|
|
191
191
|
|
|
192
192
|
def parse_from_state(self, state, last_token=None):
|
|
193
193
|
try:
|
|
194
|
-
token = last_token
|
|
195
194
|
lexer = state.lexer
|
|
195
|
+
compiled = state.parse_conf._compiled
|
|
196
|
+
|
|
197
|
+
if (last_token is None
|
|
198
|
+
and hasattr(lexer, 'state')
|
|
199
|
+
and isinstance(lexer.lexer, BasicLexer)):
|
|
200
|
+
# Fast path: LexerThread with BasicLexer — run entire
|
|
201
|
+
# lex+parse loop in a single Rust call.
|
|
202
|
+
return compiled.parse_loop(
|
|
203
|
+
lexer.lexer, lexer.state,
|
|
204
|
+
state.state_stack, state.value_stack,
|
|
205
|
+
state,
|
|
206
|
+
)
|
|
196
207
|
|
|
197
|
-
#
|
|
198
|
-
|
|
208
|
+
# Fallback: ContextualLexer or resume with last_token
|
|
209
|
+
token = last_token
|
|
199
210
|
if hasattr(lexer, 'state'):
|
|
200
|
-
# LexerThread path
|
|
201
211
|
inner_lexer = lexer.lexer
|
|
202
212
|
lexer_state = lexer.state
|
|
203
213
|
try:
|
|
@@ -207,8 +217,6 @@ class _Parser:
|
|
|
207
217
|
except EOFError:
|
|
208
218
|
pass
|
|
209
219
|
else:
|
|
210
|
-
# ContextualLexer path — uses lex() which needs
|
|
211
|
-
# lexer_state + parser_state together
|
|
212
220
|
for token in lexer.lex(state):
|
|
213
221
|
assert token is not None
|
|
214
222
|
state.feed_token(token)
|
|
@@ -2,6 +2,10 @@ use pyo3::prelude::*;
|
|
|
2
2
|
use pyo3::types::{PyDict, PyList, PySet, PyTuple};
|
|
3
3
|
use std::collections::HashMap;
|
|
4
4
|
|
|
5
|
+
use crate::lexer::BasicLexer;
|
|
6
|
+
use crate::lexer_state::LexerState;
|
|
7
|
+
use crate::token::Token;
|
|
8
|
+
|
|
5
9
|
/// A pre-compiled rule: stores the expansion size and origin name
|
|
6
10
|
/// so we never touch Python Rule objects during parsing.
|
|
7
11
|
struct CompiledRule {
|
|
@@ -143,6 +147,7 @@ impl CompiledParseTable {
|
|
|
143
147
|
/// Takes state_stack and value_stack as mutable Python lists,
|
|
144
148
|
/// the token, and is_end flag.
|
|
145
149
|
/// Returns None on shift, or the final value on successful end.
|
|
150
|
+
#[allow(clippy::too_many_arguments)]
|
|
146
151
|
fn feed_token(
|
|
147
152
|
&self,
|
|
148
153
|
state_stack: &Bound<'_, PyList>,
|
|
@@ -152,6 +157,109 @@ impl CompiledParseTable {
|
|
|
152
157
|
is_end: bool,
|
|
153
158
|
parser_state: &Bound<'_, PyAny>,
|
|
154
159
|
py: Python<'_>,
|
|
160
|
+
) -> PyResult<Option<Py<PyAny>>> {
|
|
161
|
+
self.feed_token_inner(
|
|
162
|
+
state_stack,
|
|
163
|
+
value_stack,
|
|
164
|
+
token,
|
|
165
|
+
token_type,
|
|
166
|
+
is_end,
|
|
167
|
+
parser_state,
|
|
168
|
+
py,
|
|
169
|
+
)
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
/// Run the full lex-parse loop in Rust for the BasicLexer path.
|
|
173
|
+
/// Calls next_token and feed_token in a tight Rust loop, only
|
|
174
|
+
/// crossing back to Python for callbacks.
|
|
175
|
+
#[allow(clippy::too_many_arguments)]
|
|
176
|
+
fn parse_loop(
|
|
177
|
+
&self,
|
|
178
|
+
lexer_obj: &Bound<'_, PyAny>,
|
|
179
|
+
lexer_state_obj: &Bound<'_, PyAny>,
|
|
180
|
+
state_stack: &Bound<'_, PyList>,
|
|
181
|
+
value_stack: &Bound<'_, PyList>,
|
|
182
|
+
parser_state: &Bound<'_, PyAny>,
|
|
183
|
+
py: Python<'_>,
|
|
184
|
+
) -> PyResult<Py<PyAny>> {
|
|
185
|
+
let mut last_token: Option<Token> = None;
|
|
186
|
+
|
|
187
|
+
// Tight lex-parse loop: next_token + feed_token without Python overhead
|
|
188
|
+
loop {
|
|
189
|
+
let result = {
|
|
190
|
+
let mut lexer = lexer_obj.downcast::<BasicLexer>()?.borrow_mut();
|
|
191
|
+
let mut lexer_state = lexer_state_obj.downcast::<LexerState>()?.borrow_mut();
|
|
192
|
+
lexer.next_token(&mut lexer_state, parser_state, py)
|
|
193
|
+
};
|
|
194
|
+
match result {
|
|
195
|
+
Ok(token) => {
|
|
196
|
+
let token_type = token.type_.clone();
|
|
197
|
+
let token_py = token.clone().into_pyobject(py)?.into_any();
|
|
198
|
+
last_token = Some(token);
|
|
199
|
+
self.feed_token_inner(
|
|
200
|
+
state_stack,
|
|
201
|
+
value_stack,
|
|
202
|
+
&token_py,
|
|
203
|
+
&token_type,
|
|
204
|
+
false,
|
|
205
|
+
parser_state,
|
|
206
|
+
py,
|
|
207
|
+
)?;
|
|
208
|
+
}
|
|
209
|
+
Err(e) if e.is_instance_of::<pyo3::exceptions::PyEOFError>(py) => {
|
|
210
|
+
break;
|
|
211
|
+
}
|
|
212
|
+
Err(e) => {
|
|
213
|
+
return Err(e);
|
|
214
|
+
}
|
|
215
|
+
}
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
// Feed $END token
|
|
219
|
+
let end_token = if let Some(ref lt) = last_token {
|
|
220
|
+
Token::new(
|
|
221
|
+
"$END".to_string(),
|
|
222
|
+
String::new(),
|
|
223
|
+
lt.start_pos,
|
|
224
|
+
lt.line,
|
|
225
|
+
lt.column,
|
|
226
|
+
lt.end_line,
|
|
227
|
+
lt.end_column,
|
|
228
|
+
lt.end_pos,
|
|
229
|
+
)
|
|
230
|
+
} else {
|
|
231
|
+
Token::new("$END".to_string(), String::new(), 0, 1, 1, None, None, None)
|
|
232
|
+
};
|
|
233
|
+
let end_token_py = end_token.into_pyobject(py)?.into_any();
|
|
234
|
+
|
|
235
|
+
let result = self.feed_token_inner(
|
|
236
|
+
state_stack,
|
|
237
|
+
value_stack,
|
|
238
|
+
&end_token_py,
|
|
239
|
+
"$END",
|
|
240
|
+
true,
|
|
241
|
+
parser_state,
|
|
242
|
+
py,
|
|
243
|
+
)?;
|
|
244
|
+
|
|
245
|
+
result.ok_or_else(|| {
|
|
246
|
+
pyo3::exceptions::PyRuntimeError::new_err("parse_loop: feed_token($END) returned None")
|
|
247
|
+
})
|
|
248
|
+
}
|
|
249
|
+
}
|
|
250
|
+
|
|
251
|
+
impl CompiledParseTable {
|
|
252
|
+
/// Shared feed_token implementation used by both feed_token and parse_loop.
|
|
253
|
+
#[allow(clippy::too_many_arguments)]
|
|
254
|
+
fn feed_token_inner(
|
|
255
|
+
&self,
|
|
256
|
+
state_stack: &Bound<'_, PyList>,
|
|
257
|
+
value_stack: &Bound<'_, PyList>,
|
|
258
|
+
token: &Bound<'_, PyAny>,
|
|
259
|
+
token_type: &str,
|
|
260
|
+
is_end: bool,
|
|
261
|
+
parser_state: &Bound<'_, PyAny>,
|
|
262
|
+
py: Python<'_>,
|
|
155
263
|
) -> PyResult<Option<Py<PyAny>>> {
|
|
156
264
|
loop {
|
|
157
265
|
let stack_len = state_stack.len();
|
|
@@ -209,7 +317,7 @@ impl CompiledParseTable {
|
|
|
209
317
|
value_stack.del_slice(start, vstack_len)?;
|
|
210
318
|
slice
|
|
211
319
|
} else {
|
|
212
|
-
PyList::empty(py)
|
|
320
|
+
PyList::empty(py)
|
|
213
321
|
};
|
|
214
322
|
|
|
215
323
|
let value = rule.callback.call1(py, (&children,))?;
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|