lark-rust 0.2.1__tar.gz → 0.2.2__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (30) hide show
  1. {lark_rust-0.2.1 → lark_rust-0.2.2}/.github/workflows/ci.yml +18 -1
  2. lark_rust-0.2.2/.github/workflows/release.yml +85 -0
  3. {lark_rust-0.2.1 → lark_rust-0.2.2}/Cargo.lock +29 -29
  4. {lark_rust-0.2.1 → lark_rust-0.2.2}/Cargo.toml +1 -1
  5. {lark_rust-0.2.1 → lark_rust-0.2.2}/PKG-INFO +1 -1
  6. {lark_rust-0.2.1 → lark_rust-0.2.2}/pyproject.toml +1 -1
  7. {lark_rust-0.2.1 → lark_rust-0.2.2}/src/lexer.rs +25 -12
  8. {lark_rust-0.2.1 → lark_rust-0.2.2}/src/lib.rs +1 -1
  9. {lark_rust-0.2.1 → lark_rust-0.2.2}/src/scanner.rs +24 -4
  10. lark_rust-0.2.2/tests/test_contextual_lexer.py +96 -0
  11. lark_rust-0.2.2/tests/test_issue_1.py +124 -0
  12. lark_rust-0.2.1/.github/workflows/release.yml +0 -76
  13. {lark_rust-0.2.1 → lark_rust-0.2.2}/.gitignore +0 -0
  14. {lark_rust-0.2.1 → lark_rust-0.2.2}/LICENSE +0 -0
  15. {lark_rust-0.2.1 → lark_rust-0.2.2}/README.md +0 -0
  16. {lark_rust-0.2.1 → lark_rust-0.2.2}/benchmarks/bench.py +0 -0
  17. {lark_rust-0.2.1 → lark_rust-0.2.2}/python/lark_rust/__init__.py +0 -0
  18. {lark_rust-0.2.1 → lark_rust-0.2.2}/src/lexer_state.rs +0 -0
  19. {lark_rust-0.2.1 → lark_rust-0.2.2}/src/lexer_thread.rs +0 -0
  20. {lark_rust-0.2.1 → lark_rust-0.2.2}/src/line_counter.rs +0 -0
  21. {lark_rust-0.2.1 → lark_rust-0.2.2}/src/parser.rs +0 -0
  22. {lark_rust-0.2.1 → lark_rust-0.2.2}/src/token.rs +0 -0
  23. {lark_rust-0.2.1 → lark_rust-0.2.2}/tests/test_cython_compat.py +0 -0
  24. {lark_rust-0.2.1 → lark_rust-0.2.2}/tests/test_integration.py +0 -0
  25. {lark_rust-0.2.1 → lark_rust-0.2.2}/tests/test_lexer.py +0 -0
  26. {lark_rust-0.2.1 → lark_rust-0.2.2}/tests/test_lexer_state.py +0 -0
  27. {lark_rust-0.2.1 → lark_rust-0.2.2}/tests/test_lexer_thread.py +0 -0
  28. {lark_rust-0.2.1 → lark_rust-0.2.2}/tests/test_line_counter.py +0 -0
  29. {lark_rust-0.2.1 → lark_rust-0.2.2}/tests/test_scanner.py +0 -0
  30. {lark_rust-0.2.1 → lark_rust-0.2.2}/tests/test_token.py +0 -0
@@ -14,6 +14,13 @@ jobs:
14
14
  matrix:
15
15
  os: [ubuntu-latest, macos-latest, windows-latest]
16
16
  python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"]
17
+ exclude:
18
+ # Python 3.9 isn't available on ubuntu-latest (26.04)
19
+ - os: ubuntu-latest
20
+ python-version: "3.9"
21
+ include:
22
+ - os: ubuntu-24.04
23
+ python-version: "3.9"
17
24
  steps:
18
25
  - uses: actions/checkout@v4
19
26
  - uses: actions/setup-python@v5
@@ -21,11 +28,20 @@ jobs:
21
28
  python-version: ${{ matrix.python-version }}
22
29
  - name: Install Rust
23
30
  uses: dtolnay/rust-toolchain@stable
31
+ - uses: Swatinem/rust-cache@v2
24
32
  - name: Build and install
33
+ # Full LTO is only needed for release wheels; skip it to speed up CI.
34
+ env:
35
+ CARGO_PROFILE_RELEASE_LTO: "false"
36
+ CARGO_PROFILE_RELEASE_CODEGEN_UNITS: "16"
37
+ # bash with -e so a failed build stops here on Windows too, and
38
+ # --no-index so we never fall back to the PyPI release.
39
+ shell: bash
25
40
  run: |
41
+ set -e
26
42
  pip install maturin lark pytest
27
43
  maturin build --release
28
- pip install --find-links target/wheels lark-rust
44
+ pip install --no-index --find-links target/wheels lark-rust
29
45
  - name: Run tests
30
46
  run: pytest tests/ -v
31
47
 
@@ -37,6 +53,7 @@ jobs:
37
53
  uses: dtolnay/rust-toolchain@stable
38
54
  with:
39
55
  components: clippy, rustfmt
56
+ - uses: Swatinem/rust-cache@v2
40
57
  - name: Cargo clippy
41
58
  run: cargo clippy --all-targets -- -D warnings
42
59
  - name: Cargo fmt check
@@ -0,0 +1,85 @@
1
+ name: Release
2
+
3
+ on:
4
+ push:
5
+ tags:
6
+ - "v*"
7
+ workflow_dispatch:
8
+
9
+ permissions:
10
+ contents: read
11
+
12
+ jobs:
13
+ build-sdist:
14
+ runs-on: ubuntu-latest
15
+ steps:
16
+ - uses: actions/checkout@v4
17
+ - name: Build sdist
18
+ uses: PyO3/maturin-action@v1
19
+ with:
20
+ command: sdist
21
+ args: --out dist
22
+ - uses: actions/upload-artifact@v4
23
+ with:
24
+ name: sdist
25
+ path: dist/*.tar.gz
26
+
27
+ build-wheels:
28
+ name: build-wheels (${{ matrix.platform.target }}, ${{ matrix.label || matrix.python }})
29
+ runs-on: ${{ matrix.platform.os }}
30
+ strategy:
31
+ fail-fast: false
32
+ # One interpreter per job so builds run in parallel.
33
+ # Interpreters are listed explicitly: pyo3 0.25 supports up to 3.14,
34
+ # so --find-interpreter fails once runners ship a newer Python.
35
+ matrix:
36
+ platform:
37
+ - { os: ubuntu-latest, target: x86_64 }
38
+ - { os: ubuntu-latest, target: aarch64 }
39
+ # macOS universal2 (x86_64 + arm64 fat binary)
40
+ - { os: macos-latest, target: universal2-apple-darwin }
41
+ - { os: windows-latest, target: x64 }
42
+ python: ["3.10", "3.11", "3.12", "3.13", "3.14"]
43
+ include:
44
+ # Extra Linux-only interpreters
45
+ - platform: { os: ubuntu-latest, target: x86_64 }
46
+ python: 3.8 3.9 3.14t pypy3.9 pypy3.10 pypy3.11
47
+ label: extras
48
+ - platform: { os: ubuntu-latest, target: aarch64 }
49
+ python: 3.8 3.9 3.14t pypy3.9 pypy3.10 pypy3.11
50
+ label: extras
51
+ steps:
52
+ - uses: actions/checkout@v4
53
+ - uses: actions/setup-python@v5
54
+ if: runner.os != 'Linux'
55
+ with:
56
+ python-version: ${{ matrix.python }}
57
+ - name: Build wheels
58
+ uses: PyO3/maturin-action@v1
59
+ with:
60
+ target: ${{ matrix.platform.target }}
61
+ args: --release --out dist --interpreter ${{ matrix.python }}
62
+ sccache: "true"
63
+ manylinux: auto
64
+ - uses: actions/upload-artifact@v4
65
+ with:
66
+ name: wheels-${{ matrix.platform.os }}-${{ matrix.platform.target }}-${{ matrix.label || matrix.python }}
67
+ path: dist/*.whl
68
+
69
+ publish:
70
+ needs: [build-sdist, build-wheels]
71
+ if: startsWith(github.ref, 'refs/tags/')
72
+ runs-on: ubuntu-latest
73
+ environment: pypi
74
+ permissions:
75
+ id-token: write
76
+ steps:
77
+ - uses: actions/download-artifact@v4
78
+ with:
79
+ path: dist
80
+ merge-multiple: true
81
+ - name: Publish to PyPI
82
+ uses: PyO3/maturin-action@v1
83
+ with:
84
+ command: upload
85
+ args: --non-interactive --skip-existing dist/*
@@ -4,18 +4,18 @@ version = 4
4
4
 
5
5
  [[package]]
6
6
  name = "aho-corasick"
7
- version = "1.1.4"
7
+ version = "1.1.5"
8
8
  source = "registry+https://github.com/rust-lang/crates.io-index"
9
- checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301"
9
+ checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba"
10
10
  dependencies = [
11
11
  "memchr",
12
12
  ]
13
13
 
14
14
  [[package]]
15
15
  name = "autocfg"
16
- version = "1.5.0"
16
+ version = "1.5.1"
17
17
  source = "registry+https://github.com/rust-lang/crates.io-index"
18
- checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8"
18
+ checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53"
19
19
 
20
20
  [[package]]
21
21
  name = "bit-set"
@@ -51,16 +51,16 @@ checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"
51
51
 
52
52
  [[package]]
53
53
  name = "indoc"
54
- version = "2.0.7"
54
+ version = "2.0.8"
55
55
  source = "registry+https://github.com/rust-lang/crates.io-index"
56
- checksum = "79cf5c93f93228cf8efb3ba362535fb11199ac548a09ce117c9b1adc3030d706"
56
+ checksum = "a37b2691796cffeb8a8cd305ac66e65841559f147f4e63231d0eafa4db5384d1"
57
57
  dependencies = [
58
58
  "rustversion",
59
59
  ]
60
60
 
61
61
  [[package]]
62
62
  name = "lark-rust"
63
- version = "0.2.1"
63
+ version = "0.2.2"
64
64
  dependencies = [
65
65
  "fancy-regex",
66
66
  "pyo3",
@@ -68,15 +68,15 @@ dependencies = [
68
68
 
69
69
  [[package]]
70
70
  name = "libc"
71
- version = "0.2.183"
71
+ version = "0.2.191"
72
72
  source = "registry+https://github.com/rust-lang/crates.io-index"
73
- checksum = "b5b646652bf6661599e1da8901b3b9522896f01e736bad5f723fe7a3a27f899d"
73
+ checksum = "65cd87c9a6d60536580b316ad985726104730a5151c99845f2490428ef91240a"
74
74
 
75
75
  [[package]]
76
76
  name = "memchr"
77
- version = "2.8.0"
77
+ version = "2.8.3"
78
78
  source = "registry+https://github.com/rust-lang/crates.io-index"
79
- checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79"
79
+ checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98"
80
80
 
81
81
  [[package]]
82
82
  name = "memoffset"
@@ -95,15 +95,15 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50"
95
95
 
96
96
  [[package]]
97
97
  name = "portable-atomic"
98
- version = "1.13.1"
98
+ version = "1.15.0"
99
99
  source = "registry+https://github.com/rust-lang/crates.io-index"
100
- checksum = "c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49"
100
+ checksum = "05c8b63e8d9609db387f0324918f81d68fe27748f084ef092fb35954d0539a85"
101
101
 
102
102
  [[package]]
103
103
  name = "proc-macro2"
104
- version = "1.0.106"
104
+ version = "1.0.107"
105
105
  source = "registry+https://github.com/rust-lang/crates.io-index"
106
- checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934"
106
+ checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9"
107
107
  dependencies = [
108
108
  "unicode-ident",
109
109
  ]
@@ -172,18 +172,18 @@ dependencies = [
172
172
 
173
173
  [[package]]
174
174
  name = "quote"
175
- version = "1.0.45"
175
+ version = "1.0.47"
176
176
  source = "registry+https://github.com/rust-lang/crates.io-index"
177
- checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924"
177
+ checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001"
178
178
  dependencies = [
179
179
  "proc-macro2",
180
180
  ]
181
181
 
182
182
  [[package]]
183
183
  name = "regex-automata"
184
- version = "0.4.14"
184
+ version = "0.4.19"
185
185
  source = "registry+https://github.com/rust-lang/crates.io-index"
186
- checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f"
186
+ checksum = "41d60ef07df457506a0a1a2073112eb4aae5733dbd2813c3da77d1b0fd441158"
187
187
  dependencies = [
188
188
  "aho-corasick",
189
189
  "memchr",
@@ -192,21 +192,21 @@ dependencies = [
192
192
 
193
193
  [[package]]
194
194
  name = "regex-syntax"
195
- version = "0.8.10"
195
+ version = "0.8.12"
196
196
  source = "registry+https://github.com/rust-lang/crates.io-index"
197
- checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a"
197
+ checksum = "d902046e6a28a1e4c3cab97258a0ce11e994903f01c2d97a9447a0b3df85f4f6"
198
198
 
199
199
  [[package]]
200
200
  name = "rustversion"
201
- version = "1.0.22"
201
+ version = "1.0.23"
202
202
  source = "registry+https://github.com/rust-lang/crates.io-index"
203
- checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d"
203
+ checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f"
204
204
 
205
205
  [[package]]
206
206
  name = "syn"
207
- version = "2.0.117"
207
+ version = "2.0.119"
208
208
  source = "registry+https://github.com/rust-lang/crates.io-index"
209
- checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99"
209
+ checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297"
210
210
  dependencies = [
211
211
  "proc-macro2",
212
212
  "quote",
@@ -215,15 +215,15 @@ dependencies = [
215
215
 
216
216
  [[package]]
217
217
  name = "target-lexicon"
218
- version = "0.13.5"
218
+ version = "0.13.6"
219
219
  source = "registry+https://github.com/rust-lang/crates.io-index"
220
- checksum = "adb6935a6f5c20170eeceb1a3835a49e12e19d792f6dd344ccc76a985ca5a6ca"
220
+ checksum = "8c447be3a2abfa84b2d91680c56600bd6b2873866de27b36ec911be705e291da"
221
221
 
222
222
  [[package]]
223
223
  name = "unicode-ident"
224
- version = "1.0.24"
224
+ version = "1.0.27"
225
225
  source = "registry+https://github.com/rust-lang/crates.io-index"
226
- checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75"
226
+ checksum = "a2c754d6c33795a1c324727428e5a7dedb5b06195f9890bdbcba760d3e246563"
227
227
 
228
228
  [[package]]
229
229
  name = "unindent"
@@ -1,6 +1,6 @@
1
1
  [package]
2
2
  name = "lark-rust"
3
- version = "0.2.1"
3
+ version = "0.2.2"
4
4
  edition = "2021"
5
5
  readme = "README.md"
6
6
 
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: lark-rust
3
- Version: 0.2.1
3
+ Version: 0.2.2
4
4
  Classifier: Development Status :: 4 - Beta
5
5
  Classifier: Intended Audience :: Developers
6
6
  Classifier: License :: OSI Approved :: MIT License
@@ -4,7 +4,7 @@ build-backend = "maturin"
4
4
 
5
5
  [project]
6
6
  name = "lark-rust"
7
- version = "0.2.1"
7
+ version = "0.2.2"
8
8
  description = "High-performance Rust-accelerated drop-in replacement for lark's LALR parser stack"
9
9
  requires-python = ">=3.8"
10
10
  license = "MIT"
@@ -10,6 +10,12 @@ pub struct TerminalDef {
10
10
  pub name: String,
11
11
  pub pattern: String,
12
12
  pub priority: i32,
13
+ /// Sort keys matching lark's `BasicLexer` ordering.
14
+ pub max_width: u64,
15
+ pub value_len: usize,
16
+ /// The original lark `TerminalDef`, passed to `_create_unless` so that
17
+ /// `PatternStr`/`PatternRE` distinctions and flags are preserved.
18
+ pub py_def: Py<PyAny>,
13
19
  }
14
20
 
15
21
  fn pattern_has_newline(pattern: &str) -> bool {
@@ -39,10 +45,19 @@ fn extract_terminals(conf: &Bound<'_, PyAny>, py: Python<'_>) -> PyResult<Vec<Te
39
45
  let regex_result: Py<PyAny> = to_regexp_fn.call0()?.into();
40
46
  let pattern_str: String = regex_result.extract(py)?;
41
47
  let priority: i32 = t.getattr("priority")?.extract()?;
48
+ // max_width may be sre's MAXREPEAT; saturate rather than fail.
49
+ let max_width: u64 = pattern_obj
50
+ .getattr("max_width")?
51
+ .extract()
52
+ .unwrap_or(u64::MAX);
53
+ let value_len: usize = pattern_obj.getattr("value")?.len()?;
42
54
  terminals.push(TerminalDef {
43
55
  name,
44
56
  pattern: pattern_str,
45
57
  priority,
58
+ max_width,
59
+ value_len,
60
+ py_def: t.unbind(),
46
61
  });
47
62
  }
48
63
  Ok(terminals)
@@ -78,6 +93,7 @@ pub struct BasicLexer {
78
93
  re_module: Py<PyAny>,
79
94
  g_regex_flags: i32,
80
95
  use_bytes: bool,
96
+ #[pyo3(get)]
81
97
  terminals_by_name: Py<PyAny>,
82
98
  }
83
99
 
@@ -120,10 +136,13 @@ impl BasicLexer {
120
136
  }
121
137
 
122
138
  let mut terminals_sorted = terminals;
139
+ // Same order as lark: (-priority, -max_width, -len(value), name)
123
140
  terminals_sorted.sort_by(|a, b| {
124
- let a_len = a.pattern.len() as i32;
125
- let b_len = b.pattern.len() as i32;
126
- (-a.priority, -b_len, &a.name).cmp(&(-b.priority, -a_len, &b.name))
141
+ b.priority
142
+ .cmp(&a.priority)
143
+ .then(b.max_width.cmp(&a.max_width))
144
+ .then(b.value_len.cmp(&a.value_len))
145
+ .then(a.name.cmp(&b.name))
127
146
  });
128
147
 
129
148
  let newline_types: HashSet<String> = terminals_sorted
@@ -158,15 +177,9 @@ impl BasicLexer {
158
177
  let lark_lexer = py.import("lark.lexer")?;
159
178
  let create_unless_fn = lark_lexer.getattr("_create_unless")?;
160
179
  let call_chain_cls = lark_lexer.getattr("CallChain")?;
161
- let pattern_re_cls = lark_lexer.getattr("PatternRE")?;
162
- let terminal_def_cls = lark_lexer.getattr("TerminalDef")?;
163
-
164
- let py_terminals = pyo3::types::PyList::empty(py);
165
- for t in &self.terminals {
166
- let pattern_obj = pattern_re_cls.call1((&t.pattern,))?;
167
- let td = terminal_def_cls.call1((&t.name, pattern_obj, t.priority))?;
168
- py_terminals.append(td)?;
169
- }
180
+
181
+ let py_terminals =
182
+ pyo3::types::PyList::new(py, self.terminals.iter().map(|t| t.py_def.bind(py)))?;
170
183
 
171
184
  let result = create_unless_fn.call1((
172
185
  py_terminals,
@@ -29,7 +29,7 @@ fn lark_rust(m: &Bound<'_, PyModule>) -> PyResult<()> {
29
29
  plugins.set_item("BasicLexer", m.getattr("BasicLexer")?)?;
30
30
  plugins.set_item("LexerThread", m.getattr("LexerThread")?)?;
31
31
  m.add("plugins", plugins)?;
32
- m.add("__version__", "0.1.1")?;
32
+ m.add("__version__", "0.2.2")?;
33
33
 
34
34
  Ok(())
35
35
  }
@@ -6,6 +6,9 @@ pub struct Scanner {
6
6
  combined_re: Option<Regex>,
7
7
  individual_patterns: Vec<(String, Regex)>,
8
8
  group_to_terminal: Vec<String>,
9
+ /// Capture-group index of each terminal in `combined_re`. Terminal
10
+ /// patterns may contain their own groups, so indices are not contiguous.
11
+ group_indices: Vec<usize>,
9
12
  }
10
13
 
11
14
  impl Scanner {
@@ -27,16 +30,33 @@ impl Scanner {
27
30
  if let Ok(re) = Regex::new(pattern) {
28
31
  individual_patterns.push((name.clone(), re));
29
32
  }
30
- pattern_parts.push(format!("({})", pattern));
33
+ pattern_parts.push(format!("(?P<{}>{})", name, pattern));
31
34
  group_to_terminal.push(name.clone());
32
35
  }
33
36
 
34
37
  let combined_re = Regex::new(&pattern_parts.join("|")).ok();
35
38
 
39
+ let group_indices = match combined_re {
40
+ Some(ref re) => {
41
+ let names: Vec<Option<&str>> = re.capture_names().collect();
42
+ group_to_terminal
43
+ .iter()
44
+ .map(|n| {
45
+ names
46
+ .iter()
47
+ .position(|g| *g == Some(n.as_str()))
48
+ .unwrap_or(0)
49
+ })
50
+ .collect()
51
+ }
52
+ None => Vec::new(),
53
+ };
54
+
36
55
  Ok(Scanner {
37
56
  combined_re,
38
57
  individual_patterns,
39
58
  group_to_terminal,
59
+ group_indices,
40
60
  })
41
61
  }
42
62
 
@@ -54,9 +74,9 @@ impl Scanner {
54
74
  if full.start() == 0 {
55
75
  let match_end = full.end();
56
76
  let matched = &text[pos..pos + match_end];
57
- // Find which group matched (groups are 1-indexed)
58
- for (i, name) in self.group_to_terminal.iter().enumerate() {
59
- if caps.get(i + 1).is_some_and(|m| m.start() == 0) {
77
+ // Find which terminal's group matched
78
+ for (name, &idx) in self.group_to_terminal.iter().zip(&self.group_indices) {
79
+ if caps.get(idx).is_some_and(|m| m.start() == 0) {
60
80
  return Some((matched, name.clone()));
61
81
  }
62
82
  }
@@ -0,0 +1,96 @@
1
+ """Regression tests for https://github.com/robertozimek/lark-rust/issues/1"""
2
+ import pytest
3
+ from lark import Lark
4
+ from lark.exceptions import UnexpectedCharacters, UnexpectedToken
5
+ import lark_rust
6
+
7
+ GRAMMAR = r"""
8
+ INT_NUMBER: /\d+/
9
+ FLOAT_NUMBER: /\d*\.\d+|\d+e(\+|\-)\d+/
10
+ VARIABLE: /[a-zA-Z_]\w*/
11
+ NEWLINE: "\n"
12
+ PLUS: "+"
13
+ EQ: "="
14
+ LPAR: "("
15
+ RPAR: ")"
16
+ LBRACK: "["
17
+ RBRACK: "]"
18
+ COMMA: ","
19
+ TYPE_COMPLEX: "complex"
20
+
21
+ %ignore /[ \t]+/
22
+
23
+ start: (NEWLINE* instruction)+ NEWLINE*
24
+
25
+ instruction: expr
26
+ | VARIABLE EQ expr
27
+
28
+ expr: expr PLUS atom
29
+ | atom
30
+
31
+ atom: INT_NUMBER
32
+ | FLOAT_NUMBER
33
+ | VARIABLE
34
+ | TYPE_COMPLEX LPAR expr COMMA expr RPAR
35
+ | LBRACK expr RBRACK
36
+ """
37
+
38
+
39
+ def _parsers():
40
+ kw = dict(parser="lalr", lexer="contextual")
41
+ return Lark(GRAMMAR, **kw), Lark(GRAMMAR, _plugins=lark_rust.plugins, **kw)
42
+
43
+
44
+ def _tokens(tree):
45
+ return [(t.type, t.value) for t in tree.scan_values(lambda _: True)]
46
+
47
+
48
+ @pytest.mark.parametrize("text", [
49
+ "complex_number = complex(2,3)",
50
+ "complexity + complex(1, 2.5)",
51
+ "x = 1e+5 + [y]",
52
+ "a = 1\nb = a + 2\n",
53
+ ])
54
+ def test_matches_lark(text):
55
+ ref, rust = _parsers()
56
+ assert _tokens(rust.parse(text)) == _tokens(ref.parse(text))
57
+
58
+
59
+ def test_keyword_prefix_lexed_as_variable():
60
+ _, rust = _parsers()
61
+ tokens = _tokens(rust.parse("complex_number = complex(2,3)"))
62
+ assert tokens[0] == ("VARIABLE", "complex_number")
63
+ assert tokens[2] == ("TYPE_COMPLEX", "complex")
64
+
65
+
66
+ def test_terminal_with_inner_group_does_not_shift_types():
67
+ # FLOAT_NUMBER contains a capture group; tokens after it must keep their types.
68
+ _, rust = _parsers()
69
+ tokens = _tokens(rust.parse("x = 1e+5 + [y]"))
70
+ assert ("FLOAT_NUMBER", "1e+5") in tokens
71
+ assert ("LBRACK", "[") in tokens
72
+ assert ("EQ", "=") in tokens
73
+
74
+
75
+ def test_basic_lexer_exposes_terminals_by_name():
76
+ _, rust = _parsers()
77
+ root = rust.parser.lexer.root_lexer
78
+ assert "VARIABLE" in root.terminals_by_name
79
+
80
+
81
+ def test_unexpected_token_from_root_lexer():
82
+ # "=" is valid for the root lexer but not after EQ, so the contextual
83
+ # lexer raises UnexpectedToken built from root_lexer.terminals_by_name.
84
+ ref, rust = _parsers()
85
+ with pytest.raises(UnexpectedToken) as ref_exc:
86
+ ref.parse("x = = 1")
87
+ with pytest.raises(UnexpectedToken) as rust_exc:
88
+ rust.parse("x = = 1")
89
+ assert rust_exc.value.token.type == ref_exc.value.token.type == "EQ"
90
+ assert rust_exc.value.expected == ref_exc.value.expected
91
+
92
+
93
+ def test_unexpected_characters():
94
+ _, rust = _parsers()
95
+ with pytest.raises(UnexpectedCharacters):
96
+ rust.parse("x = 1 $ 2")
@@ -0,0 +1,124 @@
1
+ """Exact reproduction from https://github.com/robertozimek/lark-rust/issues/1"""
2
+ from lark import Lark
3
+ import lark_rust
4
+
5
+ GRAMMAR = r"""// ==================== TOKENS ====================
6
+ INT_NUMBER: /\d+/
7
+ FLOAT_NUMBER: /\d*\.\d+|\d+e(\+|\-)\d+/
8
+ VARIABLE: /[a-zA-Z_]\w*/
9
+ NEWLINE: "\n"
10
+ PLUS: "+"
11
+ MINUS: "-"
12
+ MUL: "*"
13
+ DIV: "/"
14
+ MOD: "%"
15
+ POWER: "**"
16
+ EQ: "="
17
+ LPAR: "("
18
+ RPAR: ")"
19
+ LBRACK: "["
20
+ RBRACK: "]"
21
+ COMMA: ","
22
+ COLON: ":"
23
+ TYPE_COMPLEX: "complex"
24
+ TYPE_FLOAT: "float"
25
+ TYPE_INT: "int"
26
+ TYPE_VECTOR: "vector"
27
+ SUM_KEYWORD: "sum"
28
+ MEAN_KEYWORD: "mean"
29
+ DOT_KEYWORD: "dot"
30
+ PRINT_KEYWORD: "print"
31
+
32
+ %ignore /[ \t]+/
33
+ %ignore /\/\/.*\n/
34
+
35
+ start: vec_lang_program
36
+
37
+ vec_lang_program: vec_lang_instructions_sequence
38
+
39
+ vec_lang_instructions_sequence: (NEWLINE* vec_lang_instruction)+ NEWLINE*
40
+
41
+ vec_lang_instruction: arithmetic_expression_level_1
42
+ | function_decl
43
+ | variable_expression EQ arithmetic_expression_level_1
44
+ | PRINT_KEYWORD LPAR function_args RPAR
45
+
46
+ arithmetic_expression_level_1: arithmetic_expression_level_1 PLUS arithmetic_expression_level_2
47
+ | arithmetic_expression_level_1 MINUS arithmetic_expression_level_2
48
+ | arithmetic_expression_level_2
49
+
50
+ arithmetic_expression_level_2: arithmetic_expression_level_2 MUL arithmetic_expression_level_3
51
+ | arithmetic_expression_level_2 DIV arithmetic_expression_level_3
52
+ | arithmetic_expression_level_2 MOD arithmetic_expression_level_3
53
+ | arithmetic_expression_level_3
54
+
55
+ arithmetic_expression_level_3: arithmetic_expression_level_3 POWER arithmetic_expression_level_4
56
+ | arithmetic_expression_level_4
57
+
58
+ arithmetic_expression_level_4: number_expression
59
+ | variable_expression
60
+ | vector
61
+ | indexing
62
+ | function_call
63
+ | LPAR arithmetic_expression_level_1 RPAR
64
+
65
+ number_expression: number
66
+ | complex_number
67
+
68
+ number: INT_NUMBER
69
+ | FLOAT_NUMBER
70
+ | PLUS INT_NUMBER
71
+ | MINUS INT_NUMBER
72
+ | PLUS FLOAT_NUMBER
73
+ | MINUS FLOAT_NUMBER
74
+
75
+ complex_number: TYPE_COMPLEX LPAR number COMMA number RPAR
76
+ | number VARIABLE
77
+
78
+ variable_expression: VARIABLE
79
+
80
+ vector: LBRACK components RBRACK
81
+ | LBRACK range RBRACK
82
+ | slicing
83
+
84
+ components: arithmetic_expression_level_1
85
+ | components COMMA arithmetic_expression_level_1
86
+
87
+ range: INT_NUMBER COLON INT_NUMBER
88
+ | MINUS INT_NUMBER COLON INT_NUMBER
89
+ | INT_NUMBER COLON MINUS INT_NUMBER
90
+ | MINUS INT_NUMBER COLON MINUS INT_NUMBER
91
+
92
+ indexing: variable_expression LBRACK INT_NUMBER RBRACK
93
+ | vector LBRACK INT_NUMBER RBRACK
94
+
95
+ slicing: variable_expression LBRACK range RBRACK
96
+ | vector LBRACK range RBRACK
97
+
98
+ function_call: variable_expression LPAR function_args RPAR
99
+ | SUM_KEYWORD LPAR function_args RPAR
100
+ | MEAN_KEYWORD LPAR function_args RPAR
101
+ | DOT_KEYWORD LPAR function_args RPAR
102
+
103
+ function_args: arithmetic_expression_level_1
104
+ | function_args COMMA arithmetic_expression_level_1
105
+
106
+ function_decl: variable_expression LPAR function_decl_args RPAR EQ arithmetic_expression_level_1
107
+
108
+ function_decl_args: variable_expression COLON type
109
+ | function_decl_args COMMA variable_expression COLON type
110
+
111
+ type: TYPE_COMPLEX
112
+ | TYPE_FLOAT
113
+ | TYPE_INT
114
+ | TYPE_VECTOR
115
+ """
116
+
117
+
118
+ def test_issue_1_exact_repro():
119
+ text = 'complex_number = complex(2,3)'
120
+ parser = Lark(GRAMMAR, parser='lalr', lexer='contextual', _plugins=lark_rust.plugins)
121
+ tree = parser.parse(text)
122
+
123
+ expected = Lark(GRAMMAR, parser='lalr', lexer='contextual').parse(text)
124
+ assert tree == expected
@@ -1,76 +0,0 @@
1
- name: Release
2
-
3
- on:
4
- push:
5
- tags:
6
- - "v*"
7
-
8
- permissions:
9
- contents: read
10
-
11
- jobs:
12
- build-sdist:
13
- runs-on: ubuntu-latest
14
- steps:
15
- - uses: actions/checkout@v4
16
- - name: Build sdist
17
- uses: PyO3/maturin-action@v1
18
- with:
19
- command: sdist
20
- args: --out dist
21
- - uses: actions/upload-artifact@v4
22
- with:
23
- name: sdist
24
- path: dist/*.tar.gz
25
-
26
- build-wheels:
27
- runs-on: ${{ matrix.os }}
28
- strategy:
29
- fail-fast: false
30
- matrix:
31
- include:
32
- # Linux x86_64
33
- - os: ubuntu-latest
34
- target: x86_64
35
- # Linux aarch64
36
- - os: ubuntu-latest
37
- target: aarch64
38
- # macOS universal2 (x86_64 + arm64 fat binary)
39
- - os: macos-latest
40
- target: universal2-apple-darwin
41
- # Windows x86_64
42
- - os: windows-latest
43
- target: x64
44
- steps:
45
- - uses: actions/checkout@v4
46
- - uses: actions/setup-python@v5
47
- with:
48
- python-version: "3.x"
49
- - name: Build wheels
50
- uses: PyO3/maturin-action@v1
51
- with:
52
- target: ${{ matrix.target }}
53
- args: --release --out dist --find-interpreter
54
- sccache: "true"
55
- manylinux: auto
56
- - uses: actions/upload-artifact@v4
57
- with:
58
- name: wheels-${{ matrix.os }}-${{ matrix.target }}
59
- path: dist/*.whl
60
-
61
- publish:
62
- needs: [build-sdist, build-wheels]
63
- runs-on: ubuntu-latest
64
- environment: pypi
65
- permissions:
66
- id-token: write
67
- steps:
68
- - uses: actions/download-artifact@v4
69
- with:
70
- path: dist
71
- merge-multiple: true
72
- - name: Publish to PyPI
73
- uses: PyO3/maturin-action@v1
74
- with:
75
- command: upload
76
- args: --non-interactive --skip-existing dist/*
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes