lark-rust 0.2.0__tar.gz → 0.2.2__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (30) hide show
  1. {lark_rust-0.2.0 → lark_rust-0.2.2}/.github/workflows/ci.yml +18 -1
  2. lark_rust-0.2.2/.github/workflows/release.yml +85 -0
  3. {lark_rust-0.2.0 → lark_rust-0.2.2}/Cargo.lock +29 -29
  4. {lark_rust-0.2.0 → lark_rust-0.2.2}/Cargo.toml +1 -1
  5. {lark_rust-0.2.0 → lark_rust-0.2.2}/PKG-INFO +4 -4
  6. {lark_rust-0.2.0 → lark_rust-0.2.2}/README.md +3 -3
  7. {lark_rust-0.2.0 → lark_rust-0.2.2}/pyproject.toml +1 -1
  8. {lark_rust-0.2.0 → lark_rust-0.2.2}/python/lark_rust/__init__.py +14 -6
  9. {lark_rust-0.2.0 → lark_rust-0.2.2}/src/lexer.rs +25 -12
  10. {lark_rust-0.2.0 → lark_rust-0.2.2}/src/lib.rs +1 -1
  11. {lark_rust-0.2.0 → lark_rust-0.2.2}/src/parser.rs +109 -1
  12. {lark_rust-0.2.0 → lark_rust-0.2.2}/src/scanner.rs +24 -4
  13. lark_rust-0.2.2/tests/test_contextual_lexer.py +96 -0
  14. lark_rust-0.2.2/tests/test_issue_1.py +124 -0
  15. lark_rust-0.2.0/.github/workflows/release.yml +0 -76
  16. {lark_rust-0.2.0 → lark_rust-0.2.2}/.gitignore +0 -0
  17. {lark_rust-0.2.0 → lark_rust-0.2.2}/LICENSE +0 -0
  18. {lark_rust-0.2.0 → lark_rust-0.2.2}/benchmarks/bench.py +0 -0
  19. {lark_rust-0.2.0 → lark_rust-0.2.2}/src/lexer_state.rs +0 -0
  20. {lark_rust-0.2.0 → lark_rust-0.2.2}/src/lexer_thread.rs +0 -0
  21. {lark_rust-0.2.0 → lark_rust-0.2.2}/src/line_counter.rs +0 -0
  22. {lark_rust-0.2.0 → lark_rust-0.2.2}/src/token.rs +0 -0
  23. {lark_rust-0.2.0 → lark_rust-0.2.2}/tests/test_cython_compat.py +0 -0
  24. {lark_rust-0.2.0 → lark_rust-0.2.2}/tests/test_integration.py +0 -0
  25. {lark_rust-0.2.0 → lark_rust-0.2.2}/tests/test_lexer.py +0 -0
  26. {lark_rust-0.2.0 → lark_rust-0.2.2}/tests/test_lexer_state.py +0 -0
  27. {lark_rust-0.2.0 → lark_rust-0.2.2}/tests/test_lexer_thread.py +0 -0
  28. {lark_rust-0.2.0 → lark_rust-0.2.2}/tests/test_line_counter.py +0 -0
  29. {lark_rust-0.2.0 → lark_rust-0.2.2}/tests/test_scanner.py +0 -0
  30. {lark_rust-0.2.0 → lark_rust-0.2.2}/tests/test_token.py +0 -0
@@ -14,6 +14,13 @@ jobs:
14
14
  matrix:
15
15
  os: [ubuntu-latest, macos-latest, windows-latest]
16
16
  python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"]
17
+ exclude:
18
+ # Python 3.9 isn't available on ubuntu-latest (26.04)
19
+ - os: ubuntu-latest
20
+ python-version: "3.9"
21
+ include:
22
+ - os: ubuntu-24.04
23
+ python-version: "3.9"
17
24
  steps:
18
25
  - uses: actions/checkout@v4
19
26
  - uses: actions/setup-python@v5
@@ -21,11 +28,20 @@ jobs:
21
28
  python-version: ${{ matrix.python-version }}
22
29
  - name: Install Rust
23
30
  uses: dtolnay/rust-toolchain@stable
31
+ - uses: Swatinem/rust-cache@v2
24
32
  - name: Build and install
33
+ # Full LTO is only needed for release wheels; skip it to speed up CI.
34
+ env:
35
+ CARGO_PROFILE_RELEASE_LTO: "false"
36
+ CARGO_PROFILE_RELEASE_CODEGEN_UNITS: "16"
37
+ # bash with -e so a failed build stops here on Windows too, and
38
+ # --no-index so we never fall back to the PyPI release.
39
+ shell: bash
25
40
  run: |
41
+ set -e
26
42
  pip install maturin lark pytest
27
43
  maturin build --release
28
- pip install --find-links target/wheels lark-rust
44
+ pip install --no-index --find-links target/wheels lark-rust
29
45
  - name: Run tests
30
46
  run: pytest tests/ -v
31
47
 
@@ -37,6 +53,7 @@ jobs:
37
53
  uses: dtolnay/rust-toolchain@stable
38
54
  with:
39
55
  components: clippy, rustfmt
56
+ - uses: Swatinem/rust-cache@v2
40
57
  - name: Cargo clippy
41
58
  run: cargo clippy --all-targets -- -D warnings
42
59
  - name: Cargo fmt check
@@ -0,0 +1,85 @@
1
+ name: Release
2
+
3
+ on:
4
+ push:
5
+ tags:
6
+ - "v*"
7
+ workflow_dispatch:
8
+
9
+ permissions:
10
+ contents: read
11
+
12
+ jobs:
13
+ build-sdist:
14
+ runs-on: ubuntu-latest
15
+ steps:
16
+ - uses: actions/checkout@v4
17
+ - name: Build sdist
18
+ uses: PyO3/maturin-action@v1
19
+ with:
20
+ command: sdist
21
+ args: --out dist
22
+ - uses: actions/upload-artifact@v4
23
+ with:
24
+ name: sdist
25
+ path: dist/*.tar.gz
26
+
27
+ build-wheels:
28
+ name: build-wheels (${{ matrix.platform.target }}, ${{ matrix.label || matrix.python }})
29
+ runs-on: ${{ matrix.platform.os }}
30
+ strategy:
31
+ fail-fast: false
32
+ # One interpreter per job so builds run in parallel.
33
+ # Interpreters are listed explicitly: pyo3 0.25 supports up to 3.14,
34
+ # so --find-interpreter fails once runners ship a newer Python.
35
+ matrix:
36
+ platform:
37
+ - { os: ubuntu-latest, target: x86_64 }
38
+ - { os: ubuntu-latest, target: aarch64 }
39
+ # macOS universal2 (x86_64 + arm64 fat binary)
40
+ - { os: macos-latest, target: universal2-apple-darwin }
41
+ - { os: windows-latest, target: x64 }
42
+ python: ["3.10", "3.11", "3.12", "3.13", "3.14"]
43
+ include:
44
+ # Extra Linux-only interpreters
45
+ - platform: { os: ubuntu-latest, target: x86_64 }
46
+ python: 3.8 3.9 3.14t pypy3.9 pypy3.10 pypy3.11
47
+ label: extras
48
+ - platform: { os: ubuntu-latest, target: aarch64 }
49
+ python: 3.8 3.9 3.14t pypy3.9 pypy3.10 pypy3.11
50
+ label: extras
51
+ steps:
52
+ - uses: actions/checkout@v4
53
+ - uses: actions/setup-python@v5
54
+ if: runner.os != 'Linux'
55
+ with:
56
+ python-version: ${{ matrix.python }}
57
+ - name: Build wheels
58
+ uses: PyO3/maturin-action@v1
59
+ with:
60
+ target: ${{ matrix.platform.target }}
61
+ args: --release --out dist --interpreter ${{ matrix.python }}
62
+ sccache: "true"
63
+ manylinux: auto
64
+ - uses: actions/upload-artifact@v4
65
+ with:
66
+ name: wheels-${{ matrix.platform.os }}-${{ matrix.platform.target }}-${{ matrix.label || matrix.python }}
67
+ path: dist/*.whl
68
+
69
+ publish:
70
+ needs: [build-sdist, build-wheels]
71
+ if: startsWith(github.ref, 'refs/tags/')
72
+ runs-on: ubuntu-latest
73
+ environment: pypi
74
+ permissions:
75
+ id-token: write
76
+ steps:
77
+ - uses: actions/download-artifact@v4
78
+ with:
79
+ path: dist
80
+ merge-multiple: true
81
+ - name: Publish to PyPI
82
+ uses: PyO3/maturin-action@v1
83
+ with:
84
+ command: upload
85
+ args: --non-interactive --skip-existing dist/*
@@ -4,18 +4,18 @@ version = 4
4
4
 
5
5
  [[package]]
6
6
  name = "aho-corasick"
7
- version = "1.1.4"
7
+ version = "1.1.5"
8
8
  source = "registry+https://github.com/rust-lang/crates.io-index"
9
- checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301"
9
+ checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba"
10
10
  dependencies = [
11
11
  "memchr",
12
12
  ]
13
13
 
14
14
  [[package]]
15
15
  name = "autocfg"
16
- version = "1.5.0"
16
+ version = "1.5.1"
17
17
  source = "registry+https://github.com/rust-lang/crates.io-index"
18
- checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8"
18
+ checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53"
19
19
 
20
20
  [[package]]
21
21
  name = "bit-set"
@@ -51,16 +51,16 @@ checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"
51
51
 
52
52
  [[package]]
53
53
  name = "indoc"
54
- version = "2.0.7"
54
+ version = "2.0.8"
55
55
  source = "registry+https://github.com/rust-lang/crates.io-index"
56
- checksum = "79cf5c93f93228cf8efb3ba362535fb11199ac548a09ce117c9b1adc3030d706"
56
+ checksum = "a37b2691796cffeb8a8cd305ac66e65841559f147f4e63231d0eafa4db5384d1"
57
57
  dependencies = [
58
58
  "rustversion",
59
59
  ]
60
60
 
61
61
  [[package]]
62
62
  name = "lark-rust"
63
- version = "0.2.0"
63
+ version = "0.2.2"
64
64
  dependencies = [
65
65
  "fancy-regex",
66
66
  "pyo3",
@@ -68,15 +68,15 @@ dependencies = [
68
68
 
69
69
  [[package]]
70
70
  name = "libc"
71
- version = "0.2.183"
71
+ version = "0.2.191"
72
72
  source = "registry+https://github.com/rust-lang/crates.io-index"
73
- checksum = "b5b646652bf6661599e1da8901b3b9522896f01e736bad5f723fe7a3a27f899d"
73
+ checksum = "65cd87c9a6d60536580b316ad985726104730a5151c99845f2490428ef91240a"
74
74
 
75
75
  [[package]]
76
76
  name = "memchr"
77
- version = "2.8.0"
77
+ version = "2.8.3"
78
78
  source = "registry+https://github.com/rust-lang/crates.io-index"
79
- checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79"
79
+ checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98"
80
80
 
81
81
  [[package]]
82
82
  name = "memoffset"
@@ -95,15 +95,15 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50"
95
95
 
96
96
  [[package]]
97
97
  name = "portable-atomic"
98
- version = "1.13.1"
98
+ version = "1.15.0"
99
99
  source = "registry+https://github.com/rust-lang/crates.io-index"
100
- checksum = "c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49"
100
+ checksum = "05c8b63e8d9609db387f0324918f81d68fe27748f084ef092fb35954d0539a85"
101
101
 
102
102
  [[package]]
103
103
  name = "proc-macro2"
104
- version = "1.0.106"
104
+ version = "1.0.107"
105
105
  source = "registry+https://github.com/rust-lang/crates.io-index"
106
- checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934"
106
+ checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9"
107
107
  dependencies = [
108
108
  "unicode-ident",
109
109
  ]
@@ -172,18 +172,18 @@ dependencies = [
172
172
 
173
173
  [[package]]
174
174
  name = "quote"
175
- version = "1.0.45"
175
+ version = "1.0.47"
176
176
  source = "registry+https://github.com/rust-lang/crates.io-index"
177
- checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924"
177
+ checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001"
178
178
  dependencies = [
179
179
  "proc-macro2",
180
180
  ]
181
181
 
182
182
  [[package]]
183
183
  name = "regex-automata"
184
- version = "0.4.14"
184
+ version = "0.4.19"
185
185
  source = "registry+https://github.com/rust-lang/crates.io-index"
186
- checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f"
186
+ checksum = "41d60ef07df457506a0a1a2073112eb4aae5733dbd2813c3da77d1b0fd441158"
187
187
  dependencies = [
188
188
  "aho-corasick",
189
189
  "memchr",
@@ -192,21 +192,21 @@ dependencies = [
192
192
 
193
193
  [[package]]
194
194
  name = "regex-syntax"
195
- version = "0.8.10"
195
+ version = "0.8.12"
196
196
  source = "registry+https://github.com/rust-lang/crates.io-index"
197
- checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a"
197
+ checksum = "d902046e6a28a1e4c3cab97258a0ce11e994903f01c2d97a9447a0b3df85f4f6"
198
198
 
199
199
  [[package]]
200
200
  name = "rustversion"
201
- version = "1.0.22"
201
+ version = "1.0.23"
202
202
  source = "registry+https://github.com/rust-lang/crates.io-index"
203
- checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d"
203
+ checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f"
204
204
 
205
205
  [[package]]
206
206
  name = "syn"
207
- version = "2.0.117"
207
+ version = "2.0.119"
208
208
  source = "registry+https://github.com/rust-lang/crates.io-index"
209
- checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99"
209
+ checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297"
210
210
  dependencies = [
211
211
  "proc-macro2",
212
212
  "quote",
@@ -215,15 +215,15 @@ dependencies = [
215
215
 
216
216
  [[package]]
217
217
  name = "target-lexicon"
218
- version = "0.13.5"
218
+ version = "0.13.6"
219
219
  source = "registry+https://github.com/rust-lang/crates.io-index"
220
- checksum = "adb6935a6f5c20170eeceb1a3835a49e12e19d792f6dd344ccc76a985ca5a6ca"
220
+ checksum = "8c447be3a2abfa84b2d91680c56600bd6b2873866de27b36ec911be705e291da"
221
221
 
222
222
  [[package]]
223
223
  name = "unicode-ident"
224
- version = "1.0.24"
224
+ version = "1.0.27"
225
225
  source = "registry+https://github.com/rust-lang/crates.io-index"
226
- checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75"
226
+ checksum = "a2c754d6c33795a1c324727428e5a7dedb5b06195f9890bdbcba760d3e246563"
227
227
 
228
228
  [[package]]
229
229
  name = "unindent"
@@ -1,6 +1,6 @@
1
1
  [package]
2
2
  name = "lark-rust"
3
- version = "0.2.0"
3
+ version = "0.2.2"
4
4
  edition = "2021"
5
5
  readme = "README.md"
6
6
 
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: lark-rust
3
- Version: 0.2.0
3
+ Version: 0.2.2
4
4
  Classifier: Development Status :: 4 - Beta
5
5
  Classifier: Intended Audience :: Developers
6
6
  Classifier: License :: OSI Approved :: MIT License
@@ -56,9 +56,9 @@ Measured on Apple M4 Pro, Python 3.13, median of 20 iterations:
56
56
 
57
57
  | Workload | lark | lark-cython | lark-rust | vs lark | vs cython |
58
58
  |---|---|---|---|---|---|
59
- | JSON (3 KB nested) | 4.02 ms | 2.39 ms | **2.04 ms** | 2.0x | 1.2x |
60
- | Arithmetic (deep) | 3.22 ms | 1.98 ms | **1.29 ms** | 2.5x | 1.5x |
61
- | Lexer (10k words) | 28.66 ms | 16.50 ms | **11.44 ms** | 2.5x | 1.4x |
59
+ | JSON (3 KB nested) | 4.16 ms | 2.45 ms | **1.73 ms** | 2.4x | 1.4x |
60
+ | Arithmetic (deep) | 3.08 ms | 2.00 ms | **1.18 ms** | 2.6x | 1.7x |
61
+ | Lexer (10k words) | 28.41 ms | 16.89 ms | **10.61 ms** | 2.7x | 1.6x |
62
62
 
63
63
  Reproduce with `python benchmarks/bench.py` (install `lark-cython` for the three-way comparison).
64
64
 
@@ -26,9 +26,9 @@ Measured on Apple M4 Pro, Python 3.13, median of 20 iterations:
26
26
 
27
27
  | Workload | lark | lark-cython | lark-rust | vs lark | vs cython |
28
28
  |---|---|---|---|---|---|
29
- | JSON (3 KB nested) | 4.02 ms | 2.39 ms | **2.04 ms** | 2.0x | 1.2x |
30
- | Arithmetic (deep) | 3.22 ms | 1.98 ms | **1.29 ms** | 2.5x | 1.5x |
31
- | Lexer (10k words) | 28.66 ms | 16.50 ms | **11.44 ms** | 2.5x | 1.4x |
29
+ | JSON (3 KB nested) | 4.16 ms | 2.45 ms | **1.73 ms** | 2.4x | 1.4x |
30
+ | Arithmetic (deep) | 3.08 ms | 2.00 ms | **1.18 ms** | 2.6x | 1.7x |
31
+ | Lexer (10k words) | 28.41 ms | 16.89 ms | **10.61 ms** | 2.7x | 1.6x |
32
32
 
33
33
  Reproduce with `python benchmarks/bench.py` (install `lark-cython` for the three-way comparison).
34
34
 
@@ -4,7 +4,7 @@ build-backend = "maturin"
4
4
 
5
5
  [project]
6
6
  name = "lark-rust"
7
- version = "0.2.0"
7
+ version = "0.2.2"
8
8
  description = "High-performance Rust-accelerated drop-in replacement for lark's LALR parser stack"
9
9
  requires-python = ">=3.8"
10
10
  license = "MIT"
@@ -191,13 +191,23 @@ class _Parser:
191
191
 
192
192
  def parse_from_state(self, state, last_token=None):
193
193
  try:
194
- token = last_token
195
194
  lexer = state.lexer
195
+ compiled = state.parse_conf._compiled
196
+
197
+ if (last_token is None
198
+ and hasattr(lexer, 'state')
199
+ and isinstance(lexer.lexer, BasicLexer)):
200
+ # Fast path: LexerThread with BasicLexer — run entire
201
+ # lex+parse loop in a single Rust call.
202
+ return compiled.parse_loop(
203
+ lexer.lexer, lexer.state,
204
+ state.state_stack, state.value_stack,
205
+ state,
206
+ )
196
207
 
197
- # Inline the lexer loop: call next_token directly instead of
198
- # using the lex() generator. Avoids generator overhead.
208
+ # Fallback: ContextualLexer or resume with last_token
209
+ token = last_token
199
210
  if hasattr(lexer, 'state'):
200
- # LexerThread path
201
211
  inner_lexer = lexer.lexer
202
212
  lexer_state = lexer.state
203
213
  try:
@@ -207,8 +217,6 @@ class _Parser:
207
217
  except EOFError:
208
218
  pass
209
219
  else:
210
- # ContextualLexer path — uses lex() which needs
211
- # lexer_state + parser_state together
212
220
  for token in lexer.lex(state):
213
221
  assert token is not None
214
222
  state.feed_token(token)
@@ -10,6 +10,12 @@ pub struct TerminalDef {
10
10
  pub name: String,
11
11
  pub pattern: String,
12
12
  pub priority: i32,
13
+ /// Sort keys matching lark's `BasicLexer` ordering.
14
+ pub max_width: u64,
15
+ pub value_len: usize,
16
+ /// The original lark `TerminalDef`, passed to `_create_unless` so that
17
+ /// `PatternStr`/`PatternRE` distinctions and flags are preserved.
18
+ pub py_def: Py<PyAny>,
13
19
  }
14
20
 
15
21
  fn pattern_has_newline(pattern: &str) -> bool {
@@ -39,10 +45,19 @@ fn extract_terminals(conf: &Bound<'_, PyAny>, py: Python<'_>) -> PyResult<Vec<Te
39
45
  let regex_result: Py<PyAny> = to_regexp_fn.call0()?.into();
40
46
  let pattern_str: String = regex_result.extract(py)?;
41
47
  let priority: i32 = t.getattr("priority")?.extract()?;
48
+ // max_width may be sre's MAXREPEAT; saturate rather than fail.
49
+ let max_width: u64 = pattern_obj
50
+ .getattr("max_width")?
51
+ .extract()
52
+ .unwrap_or(u64::MAX);
53
+ let value_len: usize = pattern_obj.getattr("value")?.len()?;
42
54
  terminals.push(TerminalDef {
43
55
  name,
44
56
  pattern: pattern_str,
45
57
  priority,
58
+ max_width,
59
+ value_len,
60
+ py_def: t.unbind(),
46
61
  });
47
62
  }
48
63
  Ok(terminals)
@@ -78,6 +93,7 @@ pub struct BasicLexer {
78
93
  re_module: Py<PyAny>,
79
94
  g_regex_flags: i32,
80
95
  use_bytes: bool,
96
+ #[pyo3(get)]
81
97
  terminals_by_name: Py<PyAny>,
82
98
  }
83
99
 
@@ -120,10 +136,13 @@ impl BasicLexer {
120
136
  }
121
137
 
122
138
  let mut terminals_sorted = terminals;
139
+ // Same order as lark: (-priority, -max_width, -len(value), name)
123
140
  terminals_sorted.sort_by(|a, b| {
124
- let a_len = a.pattern.len() as i32;
125
- let b_len = b.pattern.len() as i32;
126
- (-a.priority, -b_len, &a.name).cmp(&(-b.priority, -a_len, &b.name))
141
+ b.priority
142
+ .cmp(&a.priority)
143
+ .then(b.max_width.cmp(&a.max_width))
144
+ .then(b.value_len.cmp(&a.value_len))
145
+ .then(a.name.cmp(&b.name))
127
146
  });
128
147
 
129
148
  let newline_types: HashSet<String> = terminals_sorted
@@ -158,15 +177,9 @@ impl BasicLexer {
158
177
  let lark_lexer = py.import("lark.lexer")?;
159
178
  let create_unless_fn = lark_lexer.getattr("_create_unless")?;
160
179
  let call_chain_cls = lark_lexer.getattr("CallChain")?;
161
- let pattern_re_cls = lark_lexer.getattr("PatternRE")?;
162
- let terminal_def_cls = lark_lexer.getattr("TerminalDef")?;
163
-
164
- let py_terminals = pyo3::types::PyList::empty(py);
165
- for t in &self.terminals {
166
- let pattern_obj = pattern_re_cls.call1((&t.pattern,))?;
167
- let td = terminal_def_cls.call1((&t.name, pattern_obj, t.priority))?;
168
- py_terminals.append(td)?;
169
- }
180
+
181
+ let py_terminals =
182
+ pyo3::types::PyList::new(py, self.terminals.iter().map(|t| t.py_def.bind(py)))?;
170
183
 
171
184
  let result = create_unless_fn.call1((
172
185
  py_terminals,
@@ -29,7 +29,7 @@ fn lark_rust(m: &Bound<'_, PyModule>) -> PyResult<()> {
29
29
  plugins.set_item("BasicLexer", m.getattr("BasicLexer")?)?;
30
30
  plugins.set_item("LexerThread", m.getattr("LexerThread")?)?;
31
31
  m.add("plugins", plugins)?;
32
- m.add("__version__", "0.1.1")?;
32
+ m.add("__version__", "0.2.2")?;
33
33
 
34
34
  Ok(())
35
35
  }
@@ -2,6 +2,10 @@ use pyo3::prelude::*;
2
2
  use pyo3::types::{PyDict, PyList, PySet, PyTuple};
3
3
  use std::collections::HashMap;
4
4
 
5
+ use crate::lexer::BasicLexer;
6
+ use crate::lexer_state::LexerState;
7
+ use crate::token::Token;
8
+
5
9
  /// A pre-compiled rule: stores the expansion size and origin name
6
10
  /// so we never touch Python Rule objects during parsing.
7
11
  struct CompiledRule {
@@ -143,6 +147,7 @@ impl CompiledParseTable {
143
147
  /// Takes state_stack and value_stack as mutable Python lists,
144
148
  /// the token, and is_end flag.
145
149
  /// Returns None on shift, or the final value on successful end.
150
+ #[allow(clippy::too_many_arguments)]
146
151
  fn feed_token(
147
152
  &self,
148
153
  state_stack: &Bound<'_, PyList>,
@@ -152,6 +157,109 @@ impl CompiledParseTable {
152
157
  is_end: bool,
153
158
  parser_state: &Bound<'_, PyAny>,
154
159
  py: Python<'_>,
160
+ ) -> PyResult<Option<Py<PyAny>>> {
161
+ self.feed_token_inner(
162
+ state_stack,
163
+ value_stack,
164
+ token,
165
+ token_type,
166
+ is_end,
167
+ parser_state,
168
+ py,
169
+ )
170
+ }
171
+
172
+ /// Run the full lex-parse loop in Rust for the BasicLexer path.
173
+ /// Calls next_token and feed_token in a tight Rust loop, only
174
+ /// crossing back to Python for callbacks.
175
+ #[allow(clippy::too_many_arguments)]
176
+ fn parse_loop(
177
+ &self,
178
+ lexer_obj: &Bound<'_, PyAny>,
179
+ lexer_state_obj: &Bound<'_, PyAny>,
180
+ state_stack: &Bound<'_, PyList>,
181
+ value_stack: &Bound<'_, PyList>,
182
+ parser_state: &Bound<'_, PyAny>,
183
+ py: Python<'_>,
184
+ ) -> PyResult<Py<PyAny>> {
185
+ let mut last_token: Option<Token> = None;
186
+
187
+ // Tight lex-parse loop: next_token + feed_token without Python overhead
188
+ loop {
189
+ let result = {
190
+ let mut lexer = lexer_obj.downcast::<BasicLexer>()?.borrow_mut();
191
+ let mut lexer_state = lexer_state_obj.downcast::<LexerState>()?.borrow_mut();
192
+ lexer.next_token(&mut lexer_state, parser_state, py)
193
+ };
194
+ match result {
195
+ Ok(token) => {
196
+ let token_type = token.type_.clone();
197
+ let token_py = token.clone().into_pyobject(py)?.into_any();
198
+ last_token = Some(token);
199
+ self.feed_token_inner(
200
+ state_stack,
201
+ value_stack,
202
+ &token_py,
203
+ &token_type,
204
+ false,
205
+ parser_state,
206
+ py,
207
+ )?;
208
+ }
209
+ Err(e) if e.is_instance_of::<pyo3::exceptions::PyEOFError>(py) => {
210
+ break;
211
+ }
212
+ Err(e) => {
213
+ return Err(e);
214
+ }
215
+ }
216
+ }
217
+
218
+ // Feed $END token
219
+ let end_token = if let Some(ref lt) = last_token {
220
+ Token::new(
221
+ "$END".to_string(),
222
+ String::new(),
223
+ lt.start_pos,
224
+ lt.line,
225
+ lt.column,
226
+ lt.end_line,
227
+ lt.end_column,
228
+ lt.end_pos,
229
+ )
230
+ } else {
231
+ Token::new("$END".to_string(), String::new(), 0, 1, 1, None, None, None)
232
+ };
233
+ let end_token_py = end_token.into_pyobject(py)?.into_any();
234
+
235
+ let result = self.feed_token_inner(
236
+ state_stack,
237
+ value_stack,
238
+ &end_token_py,
239
+ "$END",
240
+ true,
241
+ parser_state,
242
+ py,
243
+ )?;
244
+
245
+ result.ok_or_else(|| {
246
+ pyo3::exceptions::PyRuntimeError::new_err("parse_loop: feed_token($END) returned None")
247
+ })
248
+ }
249
+ }
250
+
251
+ impl CompiledParseTable {
252
+ /// Shared feed_token implementation used by both feed_token and parse_loop.
253
+ #[allow(clippy::too_many_arguments)]
254
+ fn feed_token_inner(
255
+ &self,
256
+ state_stack: &Bound<'_, PyList>,
257
+ value_stack: &Bound<'_, PyList>,
258
+ token: &Bound<'_, PyAny>,
259
+ token_type: &str,
260
+ is_end: bool,
261
+ parser_state: &Bound<'_, PyAny>,
262
+ py: Python<'_>,
155
263
  ) -> PyResult<Option<Py<PyAny>>> {
156
264
  loop {
157
265
  let stack_len = state_stack.len();
@@ -209,7 +317,7 @@ impl CompiledParseTable {
209
317
  value_stack.del_slice(start, vstack_len)?;
210
318
  slice
211
319
  } else {
212
- PyList::empty(py).into()
320
+ PyList::empty(py)
213
321
  };
214
322
 
215
323
  let value = rule.callback.call1(py, (&children,))?;
@@ -6,6 +6,9 @@ pub struct Scanner {
6
6
  combined_re: Option<Regex>,
7
7
  individual_patterns: Vec<(String, Regex)>,
8
8
  group_to_terminal: Vec<String>,
9
+ /// Capture-group index of each terminal in `combined_re`. Terminal
10
+ /// patterns may contain their own groups, so indices are not contiguous.
11
+ group_indices: Vec<usize>,
9
12
  }
10
13
 
11
14
  impl Scanner {
@@ -27,16 +30,33 @@ impl Scanner {
27
30
  if let Ok(re) = Regex::new(pattern) {
28
31
  individual_patterns.push((name.clone(), re));
29
32
  }
30
- pattern_parts.push(format!("({})", pattern));
33
+ pattern_parts.push(format!("(?P<{}>{})", name, pattern));
31
34
  group_to_terminal.push(name.clone());
32
35
  }
33
36
 
34
37
  let combined_re = Regex::new(&pattern_parts.join("|")).ok();
35
38
 
39
+ let group_indices = match combined_re {
40
+ Some(ref re) => {
41
+ let names: Vec<Option<&str>> = re.capture_names().collect();
42
+ group_to_terminal
43
+ .iter()
44
+ .map(|n| {
45
+ names
46
+ .iter()
47
+ .position(|g| *g == Some(n.as_str()))
48
+ .unwrap_or(0)
49
+ })
50
+ .collect()
51
+ }
52
+ None => Vec::new(),
53
+ };
54
+
36
55
  Ok(Scanner {
37
56
  combined_re,
38
57
  individual_patterns,
39
58
  group_to_terminal,
59
+ group_indices,
40
60
  })
41
61
  }
42
62
 
@@ -54,9 +74,9 @@ impl Scanner {
54
74
  if full.start() == 0 {
55
75
  let match_end = full.end();
56
76
  let matched = &text[pos..pos + match_end];
57
- // Find which group matched (groups are 1-indexed)
58
- for (i, name) in self.group_to_terminal.iter().enumerate() {
59
- if caps.get(i + 1).is_some_and(|m| m.start() == 0) {
77
+ // Find which terminal's group matched
78
+ for (name, &idx) in self.group_to_terminal.iter().zip(&self.group_indices) {
79
+ if caps.get(idx).is_some_and(|m| m.start() == 0) {
60
80
  return Some((matched, name.clone()));
61
81
  }
62
82
  }
@@ -0,0 +1,96 @@
1
+ """Regression tests for https://github.com/robertozimek/lark-rust/issues/1"""
2
+ import pytest
3
+ from lark import Lark
4
+ from lark.exceptions import UnexpectedCharacters, UnexpectedToken
5
+ import lark_rust
6
+
7
+ GRAMMAR = r"""
8
+ INT_NUMBER: /\d+/
9
+ FLOAT_NUMBER: /\d*\.\d+|\d+e(\+|\-)\d+/
10
+ VARIABLE: /[a-zA-Z_]\w*/
11
+ NEWLINE: "\n"
12
+ PLUS: "+"
13
+ EQ: "="
14
+ LPAR: "("
15
+ RPAR: ")"
16
+ LBRACK: "["
17
+ RBRACK: "]"
18
+ COMMA: ","
19
+ TYPE_COMPLEX: "complex"
20
+
21
+ %ignore /[ \t]+/
22
+
23
+ start: (NEWLINE* instruction)+ NEWLINE*
24
+
25
+ instruction: expr
26
+ | VARIABLE EQ expr
27
+
28
+ expr: expr PLUS atom
29
+ | atom
30
+
31
+ atom: INT_NUMBER
32
+ | FLOAT_NUMBER
33
+ | VARIABLE
34
+ | TYPE_COMPLEX LPAR expr COMMA expr RPAR
35
+ | LBRACK expr RBRACK
36
+ """
37
+
38
+
39
+ def _parsers():
40
+ kw = dict(parser="lalr", lexer="contextual")
41
+ return Lark(GRAMMAR, **kw), Lark(GRAMMAR, _plugins=lark_rust.plugins, **kw)
42
+
43
+
44
+ def _tokens(tree):
45
+ return [(t.type, t.value) for t in tree.scan_values(lambda _: True)]
46
+
47
+
48
+ @pytest.mark.parametrize("text", [
49
+ "complex_number = complex(2,3)",
50
+ "complexity + complex(1, 2.5)",
51
+ "x = 1e+5 + [y]",
52
+ "a = 1\nb = a + 2\n",
53
+ ])
54
+ def test_matches_lark(text):
55
+ ref, rust = _parsers()
56
+ assert _tokens(rust.parse(text)) == _tokens(ref.parse(text))
57
+
58
+
59
+ def test_keyword_prefix_lexed_as_variable():
60
+ _, rust = _parsers()
61
+ tokens = _tokens(rust.parse("complex_number = complex(2,3)"))
62
+ assert tokens[0] == ("VARIABLE", "complex_number")
63
+ assert tokens[2] == ("TYPE_COMPLEX", "complex")
64
+
65
+
66
+ def test_terminal_with_inner_group_does_not_shift_types():
67
+ # FLOAT_NUMBER contains a capture group; tokens after it must keep their types.
68
+ _, rust = _parsers()
69
+ tokens = _tokens(rust.parse("x = 1e+5 + [y]"))
70
+ assert ("FLOAT_NUMBER", "1e+5") in tokens
71
+ assert ("LBRACK", "[") in tokens
72
+ assert ("EQ", "=") in tokens
73
+
74
+
75
+ def test_basic_lexer_exposes_terminals_by_name():
76
+ _, rust = _parsers()
77
+ root = rust.parser.lexer.root_lexer
78
+ assert "VARIABLE" in root.terminals_by_name
79
+
80
+
81
+ def test_unexpected_token_from_root_lexer():
82
+ # "=" is valid for the root lexer but not after EQ, so the contextual
83
+ # lexer raises UnexpectedToken built from root_lexer.terminals_by_name.
84
+ ref, rust = _parsers()
85
+ with pytest.raises(UnexpectedToken) as ref_exc:
86
+ ref.parse("x = = 1")
87
+ with pytest.raises(UnexpectedToken) as rust_exc:
88
+ rust.parse("x = = 1")
89
+ assert rust_exc.value.token.type == ref_exc.value.token.type == "EQ"
90
+ assert rust_exc.value.expected == ref_exc.value.expected
91
+
92
+
93
+ def test_unexpected_characters():
94
+ _, rust = _parsers()
95
+ with pytest.raises(UnexpectedCharacters):
96
+ rust.parse("x = 1 $ 2")
@@ -0,0 +1,124 @@
1
+ """Exact reproduction from https://github.com/robertozimek/lark-rust/issues/1"""
2
+ from lark import Lark
3
+ import lark_rust
4
+
5
+ GRAMMAR = r"""// ==================== TOKENS ====================
6
+ INT_NUMBER: /\d+/
7
+ FLOAT_NUMBER: /\d*\.\d+|\d+e(\+|\-)\d+/
8
+ VARIABLE: /[a-zA-Z_]\w*/
9
+ NEWLINE: "\n"
10
+ PLUS: "+"
11
+ MINUS: "-"
12
+ MUL: "*"
13
+ DIV: "/"
14
+ MOD: "%"
15
+ POWER: "**"
16
+ EQ: "="
17
+ LPAR: "("
18
+ RPAR: ")"
19
+ LBRACK: "["
20
+ RBRACK: "]"
21
+ COMMA: ","
22
+ COLON: ":"
23
+ TYPE_COMPLEX: "complex"
24
+ TYPE_FLOAT: "float"
25
+ TYPE_INT: "int"
26
+ TYPE_VECTOR: "vector"
27
+ SUM_KEYWORD: "sum"
28
+ MEAN_KEYWORD: "mean"
29
+ DOT_KEYWORD: "dot"
30
+ PRINT_KEYWORD: "print"
31
+
32
+ %ignore /[ \t]+/
33
+ %ignore /\/\/.*\n/
34
+
35
+ start: vec_lang_program
36
+
37
+ vec_lang_program: vec_lang_instructions_sequence
38
+
39
+ vec_lang_instructions_sequence: (NEWLINE* vec_lang_instruction)+ NEWLINE*
40
+
41
+ vec_lang_instruction: arithmetic_expression_level_1
42
+ | function_decl
43
+ | variable_expression EQ arithmetic_expression_level_1
44
+ | PRINT_KEYWORD LPAR function_args RPAR
45
+
46
+ arithmetic_expression_level_1: arithmetic_expression_level_1 PLUS arithmetic_expression_level_2
47
+ | arithmetic_expression_level_1 MINUS arithmetic_expression_level_2
48
+ | arithmetic_expression_level_2
49
+
50
+ arithmetic_expression_level_2: arithmetic_expression_level_2 MUL arithmetic_expression_level_3
51
+ | arithmetic_expression_level_2 DIV arithmetic_expression_level_3
52
+ | arithmetic_expression_level_2 MOD arithmetic_expression_level_3
53
+ | arithmetic_expression_level_3
54
+
55
+ arithmetic_expression_level_3: arithmetic_expression_level_3 POWER arithmetic_expression_level_4
56
+ | arithmetic_expression_level_4
57
+
58
+ arithmetic_expression_level_4: number_expression
59
+ | variable_expression
60
+ | vector
61
+ | indexing
62
+ | function_call
63
+ | LPAR arithmetic_expression_level_1 RPAR
64
+
65
+ number_expression: number
66
+ | complex_number
67
+
68
+ number: INT_NUMBER
69
+ | FLOAT_NUMBER
70
+ | PLUS INT_NUMBER
71
+ | MINUS INT_NUMBER
72
+ | PLUS FLOAT_NUMBER
73
+ | MINUS FLOAT_NUMBER
74
+
75
+ complex_number: TYPE_COMPLEX LPAR number COMMA number RPAR
76
+ | number VARIABLE
77
+
78
+ variable_expression: VARIABLE
79
+
80
+ vector: LBRACK components RBRACK
81
+ | LBRACK range RBRACK
82
+ | slicing
83
+
84
+ components: arithmetic_expression_level_1
85
+ | components COMMA arithmetic_expression_level_1
86
+
87
+ range: INT_NUMBER COLON INT_NUMBER
88
+ | MINUS INT_NUMBER COLON INT_NUMBER
89
+ | INT_NUMBER COLON MINUS INT_NUMBER
90
+ | MINUS INT_NUMBER COLON MINUS INT_NUMBER
91
+
92
+ indexing: variable_expression LBRACK INT_NUMBER RBRACK
93
+ | vector LBRACK INT_NUMBER RBRACK
94
+
95
+ slicing: variable_expression LBRACK range RBRACK
96
+ | vector LBRACK range RBRACK
97
+
98
+ function_call: variable_expression LPAR function_args RPAR
99
+ | SUM_KEYWORD LPAR function_args RPAR
100
+ | MEAN_KEYWORD LPAR function_args RPAR
101
+ | DOT_KEYWORD LPAR function_args RPAR
102
+
103
+ function_args: arithmetic_expression_level_1
104
+ | function_args COMMA arithmetic_expression_level_1
105
+
106
+ function_decl: variable_expression LPAR function_decl_args RPAR EQ arithmetic_expression_level_1
107
+
108
+ function_decl_args: variable_expression COLON type
109
+ | function_decl_args COMMA variable_expression COLON type
110
+
111
+ type: TYPE_COMPLEX
112
+ | TYPE_FLOAT
113
+ | TYPE_INT
114
+ | TYPE_VECTOR
115
+ """
116
+
117
+
118
+ def test_issue_1_exact_repro():
119
+ text = 'complex_number = complex(2,3)'
120
+ parser = Lark(GRAMMAR, parser='lalr', lexer='contextual', _plugins=lark_rust.plugins)
121
+ tree = parser.parse(text)
122
+
123
+ expected = Lark(GRAMMAR, parser='lalr', lexer='contextual').parse(text)
124
+ assert tree == expected
@@ -1,76 +0,0 @@
1
- name: Release
2
-
3
- on:
4
- push:
5
- tags:
6
- - "v*"
7
-
8
- permissions:
9
- contents: read
10
-
11
- jobs:
12
- build-sdist:
13
- runs-on: ubuntu-latest
14
- steps:
15
- - uses: actions/checkout@v4
16
- - name: Build sdist
17
- uses: PyO3/maturin-action@v1
18
- with:
19
- command: sdist
20
- args: --out dist
21
- - uses: actions/upload-artifact@v4
22
- with:
23
- name: sdist
24
- path: dist/*.tar.gz
25
-
26
- build-wheels:
27
- runs-on: ${{ matrix.os }}
28
- strategy:
29
- fail-fast: false
30
- matrix:
31
- include:
32
- # Linux x86_64
33
- - os: ubuntu-latest
34
- target: x86_64
35
- # Linux aarch64
36
- - os: ubuntu-latest
37
- target: aarch64
38
- # macOS universal2 (x86_64 + arm64 fat binary)
39
- - os: macos-latest
40
- target: universal2-apple-darwin
41
- # Windows x86_64
42
- - os: windows-latest
43
- target: x64
44
- steps:
45
- - uses: actions/checkout@v4
46
- - uses: actions/setup-python@v5
47
- with:
48
- python-version: "3.x"
49
- - name: Build wheels
50
- uses: PyO3/maturin-action@v1
51
- with:
52
- target: ${{ matrix.target }}
53
- args: --release --out dist --find-interpreter
54
- sccache: "true"
55
- manylinux: auto
56
- - uses: actions/upload-artifact@v4
57
- with:
58
- name: wheels-${{ matrix.os }}-${{ matrix.target }}
59
- path: dist/*.whl
60
-
61
- publish:
62
- needs: [build-sdist, build-wheels]
63
- runs-on: ubuntu-latest
64
- environment: pypi
65
- permissions:
66
- id-token: write
67
- steps:
68
- - uses: actions/download-artifact@v4
69
- with:
70
- path: dist
71
- merge-multiple: true
72
- - name: Publish to PyPI
73
- uses: PyO3/maturin-action@v1
74
- with:
75
- command: upload
76
- args: --non-interactive --skip-existing dist/*
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes