lark-rust 0.2.1__tar.gz → 0.2.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {lark_rust-0.2.1 → lark_rust-0.2.2}/.github/workflows/ci.yml +18 -1
- lark_rust-0.2.2/.github/workflows/release.yml +85 -0
- {lark_rust-0.2.1 → lark_rust-0.2.2}/Cargo.lock +29 -29
- {lark_rust-0.2.1 → lark_rust-0.2.2}/Cargo.toml +1 -1
- {lark_rust-0.2.1 → lark_rust-0.2.2}/PKG-INFO +1 -1
- {lark_rust-0.2.1 → lark_rust-0.2.2}/pyproject.toml +1 -1
- {lark_rust-0.2.1 → lark_rust-0.2.2}/src/lexer.rs +25 -12
- {lark_rust-0.2.1 → lark_rust-0.2.2}/src/lib.rs +1 -1
- {lark_rust-0.2.1 → lark_rust-0.2.2}/src/scanner.rs +24 -4
- lark_rust-0.2.2/tests/test_contextual_lexer.py +96 -0
- lark_rust-0.2.2/tests/test_issue_1.py +124 -0
- lark_rust-0.2.1/.github/workflows/release.yml +0 -76
- {lark_rust-0.2.1 → lark_rust-0.2.2}/.gitignore +0 -0
- {lark_rust-0.2.1 → lark_rust-0.2.2}/LICENSE +0 -0
- {lark_rust-0.2.1 → lark_rust-0.2.2}/README.md +0 -0
- {lark_rust-0.2.1 → lark_rust-0.2.2}/benchmarks/bench.py +0 -0
- {lark_rust-0.2.1 → lark_rust-0.2.2}/python/lark_rust/__init__.py +0 -0
- {lark_rust-0.2.1 → lark_rust-0.2.2}/src/lexer_state.rs +0 -0
- {lark_rust-0.2.1 → lark_rust-0.2.2}/src/lexer_thread.rs +0 -0
- {lark_rust-0.2.1 → lark_rust-0.2.2}/src/line_counter.rs +0 -0
- {lark_rust-0.2.1 → lark_rust-0.2.2}/src/parser.rs +0 -0
- {lark_rust-0.2.1 → lark_rust-0.2.2}/src/token.rs +0 -0
- {lark_rust-0.2.1 → lark_rust-0.2.2}/tests/test_cython_compat.py +0 -0
- {lark_rust-0.2.1 → lark_rust-0.2.2}/tests/test_integration.py +0 -0
- {lark_rust-0.2.1 → lark_rust-0.2.2}/tests/test_lexer.py +0 -0
- {lark_rust-0.2.1 → lark_rust-0.2.2}/tests/test_lexer_state.py +0 -0
- {lark_rust-0.2.1 → lark_rust-0.2.2}/tests/test_lexer_thread.py +0 -0
- {lark_rust-0.2.1 → lark_rust-0.2.2}/tests/test_line_counter.py +0 -0
- {lark_rust-0.2.1 → lark_rust-0.2.2}/tests/test_scanner.py +0 -0
- {lark_rust-0.2.1 → lark_rust-0.2.2}/tests/test_token.py +0 -0
|
@@ -14,6 +14,13 @@ jobs:
|
|
|
14
14
|
matrix:
|
|
15
15
|
os: [ubuntu-latest, macos-latest, windows-latest]
|
|
16
16
|
python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"]
|
|
17
|
+
exclude:
|
|
18
|
+
# Python 3.9 isn't available on ubuntu-latest (26.04)
|
|
19
|
+
- os: ubuntu-latest
|
|
20
|
+
python-version: "3.9"
|
|
21
|
+
include:
|
|
22
|
+
- os: ubuntu-24.04
|
|
23
|
+
python-version: "3.9"
|
|
17
24
|
steps:
|
|
18
25
|
- uses: actions/checkout@v4
|
|
19
26
|
- uses: actions/setup-python@v5
|
|
@@ -21,11 +28,20 @@ jobs:
|
|
|
21
28
|
python-version: ${{ matrix.python-version }}
|
|
22
29
|
- name: Install Rust
|
|
23
30
|
uses: dtolnay/rust-toolchain@stable
|
|
31
|
+
- uses: Swatinem/rust-cache@v2
|
|
24
32
|
- name: Build and install
|
|
33
|
+
# Full LTO is only needed for release wheels; skip it to speed up CI.
|
|
34
|
+
env:
|
|
35
|
+
CARGO_PROFILE_RELEASE_LTO: "false"
|
|
36
|
+
CARGO_PROFILE_RELEASE_CODEGEN_UNITS: "16"
|
|
37
|
+
# bash with -e so a failed build stops here on Windows too, and
|
|
38
|
+
# --no-index so we never fall back to the PyPI release.
|
|
39
|
+
shell: bash
|
|
25
40
|
run: |
|
|
41
|
+
set -e
|
|
26
42
|
pip install maturin lark pytest
|
|
27
43
|
maturin build --release
|
|
28
|
-
pip install --find-links target/wheels lark-rust
|
|
44
|
+
pip install --no-index --find-links target/wheels lark-rust
|
|
29
45
|
- name: Run tests
|
|
30
46
|
run: pytest tests/ -v
|
|
31
47
|
|
|
@@ -37,6 +53,7 @@ jobs:
|
|
|
37
53
|
uses: dtolnay/rust-toolchain@stable
|
|
38
54
|
with:
|
|
39
55
|
components: clippy, rustfmt
|
|
56
|
+
- uses: Swatinem/rust-cache@v2
|
|
40
57
|
- name: Cargo clippy
|
|
41
58
|
run: cargo clippy --all-targets -- -D warnings
|
|
42
59
|
- name: Cargo fmt check
|
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
name: Release
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
tags:
|
|
6
|
+
- "v*"
|
|
7
|
+
workflow_dispatch:
|
|
8
|
+
|
|
9
|
+
permissions:
|
|
10
|
+
contents: read
|
|
11
|
+
|
|
12
|
+
jobs:
|
|
13
|
+
build-sdist:
|
|
14
|
+
runs-on: ubuntu-latest
|
|
15
|
+
steps:
|
|
16
|
+
- uses: actions/checkout@v4
|
|
17
|
+
- name: Build sdist
|
|
18
|
+
uses: PyO3/maturin-action@v1
|
|
19
|
+
with:
|
|
20
|
+
command: sdist
|
|
21
|
+
args: --out dist
|
|
22
|
+
- uses: actions/upload-artifact@v4
|
|
23
|
+
with:
|
|
24
|
+
name: sdist
|
|
25
|
+
path: dist/*.tar.gz
|
|
26
|
+
|
|
27
|
+
build-wheels:
|
|
28
|
+
name: build-wheels (${{ matrix.platform.target }}, ${{ matrix.label || matrix.python }})
|
|
29
|
+
runs-on: ${{ matrix.platform.os }}
|
|
30
|
+
strategy:
|
|
31
|
+
fail-fast: false
|
|
32
|
+
# One interpreter per job so builds run in parallel.
|
|
33
|
+
# Interpreters are listed explicitly: pyo3 0.25 supports up to 3.14,
|
|
34
|
+
# so --find-interpreter fails once runners ship a newer Python.
|
|
35
|
+
matrix:
|
|
36
|
+
platform:
|
|
37
|
+
- { os: ubuntu-latest, target: x86_64 }
|
|
38
|
+
- { os: ubuntu-latest, target: aarch64 }
|
|
39
|
+
# macOS universal2 (x86_64 + arm64 fat binary)
|
|
40
|
+
- { os: macos-latest, target: universal2-apple-darwin }
|
|
41
|
+
- { os: windows-latest, target: x64 }
|
|
42
|
+
python: ["3.10", "3.11", "3.12", "3.13", "3.14"]
|
|
43
|
+
include:
|
|
44
|
+
# Extra Linux-only interpreters
|
|
45
|
+
- platform: { os: ubuntu-latest, target: x86_64 }
|
|
46
|
+
python: 3.8 3.9 3.14t pypy3.9 pypy3.10 pypy3.11
|
|
47
|
+
label: extras
|
|
48
|
+
- platform: { os: ubuntu-latest, target: aarch64 }
|
|
49
|
+
python: 3.8 3.9 3.14t pypy3.9 pypy3.10 pypy3.11
|
|
50
|
+
label: extras
|
|
51
|
+
steps:
|
|
52
|
+
- uses: actions/checkout@v4
|
|
53
|
+
- uses: actions/setup-python@v5
|
|
54
|
+
if: runner.os != 'Linux'
|
|
55
|
+
with:
|
|
56
|
+
python-version: ${{ matrix.python }}
|
|
57
|
+
- name: Build wheels
|
|
58
|
+
uses: PyO3/maturin-action@v1
|
|
59
|
+
with:
|
|
60
|
+
target: ${{ matrix.platform.target }}
|
|
61
|
+
args: --release --out dist --interpreter ${{ matrix.python }}
|
|
62
|
+
sccache: "true"
|
|
63
|
+
manylinux: auto
|
|
64
|
+
- uses: actions/upload-artifact@v4
|
|
65
|
+
with:
|
|
66
|
+
name: wheels-${{ matrix.platform.os }}-${{ matrix.platform.target }}-${{ matrix.label || matrix.python }}
|
|
67
|
+
path: dist/*.whl
|
|
68
|
+
|
|
69
|
+
publish:
|
|
70
|
+
needs: [build-sdist, build-wheels]
|
|
71
|
+
if: startsWith(github.ref, 'refs/tags/')
|
|
72
|
+
runs-on: ubuntu-latest
|
|
73
|
+
environment: pypi
|
|
74
|
+
permissions:
|
|
75
|
+
id-token: write
|
|
76
|
+
steps:
|
|
77
|
+
- uses: actions/download-artifact@v4
|
|
78
|
+
with:
|
|
79
|
+
path: dist
|
|
80
|
+
merge-multiple: true
|
|
81
|
+
- name: Publish to PyPI
|
|
82
|
+
uses: PyO3/maturin-action@v1
|
|
83
|
+
with:
|
|
84
|
+
command: upload
|
|
85
|
+
args: --non-interactive --skip-existing dist/*
|
|
@@ -4,18 +4,18 @@ version = 4
|
|
|
4
4
|
|
|
5
5
|
[[package]]
|
|
6
6
|
name = "aho-corasick"
|
|
7
|
-
version = "1.1.
|
|
7
|
+
version = "1.1.5"
|
|
8
8
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
9
|
-
checksum = "
|
|
9
|
+
checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba"
|
|
10
10
|
dependencies = [
|
|
11
11
|
"memchr",
|
|
12
12
|
]
|
|
13
13
|
|
|
14
14
|
[[package]]
|
|
15
15
|
name = "autocfg"
|
|
16
|
-
version = "1.5.
|
|
16
|
+
version = "1.5.1"
|
|
17
17
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
18
|
-
checksum = "
|
|
18
|
+
checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53"
|
|
19
19
|
|
|
20
20
|
[[package]]
|
|
21
21
|
name = "bit-set"
|
|
@@ -51,16 +51,16 @@ checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"
|
|
|
51
51
|
|
|
52
52
|
[[package]]
|
|
53
53
|
name = "indoc"
|
|
54
|
-
version = "2.0.
|
|
54
|
+
version = "2.0.8"
|
|
55
55
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
56
|
-
checksum = "
|
|
56
|
+
checksum = "a37b2691796cffeb8a8cd305ac66e65841559f147f4e63231d0eafa4db5384d1"
|
|
57
57
|
dependencies = [
|
|
58
58
|
"rustversion",
|
|
59
59
|
]
|
|
60
60
|
|
|
61
61
|
[[package]]
|
|
62
62
|
name = "lark-rust"
|
|
63
|
-
version = "0.2.
|
|
63
|
+
version = "0.2.2"
|
|
64
64
|
dependencies = [
|
|
65
65
|
"fancy-regex",
|
|
66
66
|
"pyo3",
|
|
@@ -68,15 +68,15 @@ dependencies = [
|
|
|
68
68
|
|
|
69
69
|
[[package]]
|
|
70
70
|
name = "libc"
|
|
71
|
-
version = "0.2.
|
|
71
|
+
version = "0.2.191"
|
|
72
72
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
73
|
-
checksum = "
|
|
73
|
+
checksum = "65cd87c9a6d60536580b316ad985726104730a5151c99845f2490428ef91240a"
|
|
74
74
|
|
|
75
75
|
[[package]]
|
|
76
76
|
name = "memchr"
|
|
77
|
-
version = "2.8.
|
|
77
|
+
version = "2.8.3"
|
|
78
78
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
79
|
-
checksum = "
|
|
79
|
+
checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98"
|
|
80
80
|
|
|
81
81
|
[[package]]
|
|
82
82
|
name = "memoffset"
|
|
@@ -95,15 +95,15 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50"
|
|
|
95
95
|
|
|
96
96
|
[[package]]
|
|
97
97
|
name = "portable-atomic"
|
|
98
|
-
version = "1.
|
|
98
|
+
version = "1.15.0"
|
|
99
99
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
100
|
-
checksum = "
|
|
100
|
+
checksum = "05c8b63e8d9609db387f0324918f81d68fe27748f084ef092fb35954d0539a85"
|
|
101
101
|
|
|
102
102
|
[[package]]
|
|
103
103
|
name = "proc-macro2"
|
|
104
|
-
version = "1.0.
|
|
104
|
+
version = "1.0.107"
|
|
105
105
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
106
|
-
checksum = "
|
|
106
|
+
checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9"
|
|
107
107
|
dependencies = [
|
|
108
108
|
"unicode-ident",
|
|
109
109
|
]
|
|
@@ -172,18 +172,18 @@ dependencies = [
|
|
|
172
172
|
|
|
173
173
|
[[package]]
|
|
174
174
|
name = "quote"
|
|
175
|
-
version = "1.0.
|
|
175
|
+
version = "1.0.47"
|
|
176
176
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
177
|
-
checksum = "
|
|
177
|
+
checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001"
|
|
178
178
|
dependencies = [
|
|
179
179
|
"proc-macro2",
|
|
180
180
|
]
|
|
181
181
|
|
|
182
182
|
[[package]]
|
|
183
183
|
name = "regex-automata"
|
|
184
|
-
version = "0.4.
|
|
184
|
+
version = "0.4.19"
|
|
185
185
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
186
|
-
checksum = "
|
|
186
|
+
checksum = "41d60ef07df457506a0a1a2073112eb4aae5733dbd2813c3da77d1b0fd441158"
|
|
187
187
|
dependencies = [
|
|
188
188
|
"aho-corasick",
|
|
189
189
|
"memchr",
|
|
@@ -192,21 +192,21 @@ dependencies = [
|
|
|
192
192
|
|
|
193
193
|
[[package]]
|
|
194
194
|
name = "regex-syntax"
|
|
195
|
-
version = "0.8.
|
|
195
|
+
version = "0.8.12"
|
|
196
196
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
197
|
-
checksum = "
|
|
197
|
+
checksum = "d902046e6a28a1e4c3cab97258a0ce11e994903f01c2d97a9447a0b3df85f4f6"
|
|
198
198
|
|
|
199
199
|
[[package]]
|
|
200
200
|
name = "rustversion"
|
|
201
|
-
version = "1.0.
|
|
201
|
+
version = "1.0.23"
|
|
202
202
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
203
|
-
checksum = "
|
|
203
|
+
checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f"
|
|
204
204
|
|
|
205
205
|
[[package]]
|
|
206
206
|
name = "syn"
|
|
207
|
-
version = "2.0.
|
|
207
|
+
version = "2.0.119"
|
|
208
208
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
209
|
-
checksum = "
|
|
209
|
+
checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297"
|
|
210
210
|
dependencies = [
|
|
211
211
|
"proc-macro2",
|
|
212
212
|
"quote",
|
|
@@ -215,15 +215,15 @@ dependencies = [
|
|
|
215
215
|
|
|
216
216
|
[[package]]
|
|
217
217
|
name = "target-lexicon"
|
|
218
|
-
version = "0.13.
|
|
218
|
+
version = "0.13.6"
|
|
219
219
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
220
|
-
checksum = "
|
|
220
|
+
checksum = "8c447be3a2abfa84b2d91680c56600bd6b2873866de27b36ec911be705e291da"
|
|
221
221
|
|
|
222
222
|
[[package]]
|
|
223
223
|
name = "unicode-ident"
|
|
224
|
-
version = "1.0.
|
|
224
|
+
version = "1.0.27"
|
|
225
225
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
226
|
-
checksum = "
|
|
226
|
+
checksum = "a2c754d6c33795a1c324727428e5a7dedb5b06195f9890bdbcba760d3e246563"
|
|
227
227
|
|
|
228
228
|
[[package]]
|
|
229
229
|
name = "unindent"
|
|
@@ -10,6 +10,12 @@ pub struct TerminalDef {
|
|
|
10
10
|
pub name: String,
|
|
11
11
|
pub pattern: String,
|
|
12
12
|
pub priority: i32,
|
|
13
|
+
/// Sort keys matching lark's `BasicLexer` ordering.
|
|
14
|
+
pub max_width: u64,
|
|
15
|
+
pub value_len: usize,
|
|
16
|
+
/// The original lark `TerminalDef`, passed to `_create_unless` so that
|
|
17
|
+
/// `PatternStr`/`PatternRE` distinctions and flags are preserved.
|
|
18
|
+
pub py_def: Py<PyAny>,
|
|
13
19
|
}
|
|
14
20
|
|
|
15
21
|
fn pattern_has_newline(pattern: &str) -> bool {
|
|
@@ -39,10 +45,19 @@ fn extract_terminals(conf: &Bound<'_, PyAny>, py: Python<'_>) -> PyResult<Vec<Te
|
|
|
39
45
|
let regex_result: Py<PyAny> = to_regexp_fn.call0()?.into();
|
|
40
46
|
let pattern_str: String = regex_result.extract(py)?;
|
|
41
47
|
let priority: i32 = t.getattr("priority")?.extract()?;
|
|
48
|
+
// max_width may be sre's MAXREPEAT; saturate rather than fail.
|
|
49
|
+
let max_width: u64 = pattern_obj
|
|
50
|
+
.getattr("max_width")?
|
|
51
|
+
.extract()
|
|
52
|
+
.unwrap_or(u64::MAX);
|
|
53
|
+
let value_len: usize = pattern_obj.getattr("value")?.len()?;
|
|
42
54
|
terminals.push(TerminalDef {
|
|
43
55
|
name,
|
|
44
56
|
pattern: pattern_str,
|
|
45
57
|
priority,
|
|
58
|
+
max_width,
|
|
59
|
+
value_len,
|
|
60
|
+
py_def: t.unbind(),
|
|
46
61
|
});
|
|
47
62
|
}
|
|
48
63
|
Ok(terminals)
|
|
@@ -78,6 +93,7 @@ pub struct BasicLexer {
|
|
|
78
93
|
re_module: Py<PyAny>,
|
|
79
94
|
g_regex_flags: i32,
|
|
80
95
|
use_bytes: bool,
|
|
96
|
+
#[pyo3(get)]
|
|
81
97
|
terminals_by_name: Py<PyAny>,
|
|
82
98
|
}
|
|
83
99
|
|
|
@@ -120,10 +136,13 @@ impl BasicLexer {
|
|
|
120
136
|
}
|
|
121
137
|
|
|
122
138
|
let mut terminals_sorted = terminals;
|
|
139
|
+
// Same order as lark: (-priority, -max_width, -len(value), name)
|
|
123
140
|
terminals_sorted.sort_by(|a, b| {
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
141
|
+
b.priority
|
|
142
|
+
.cmp(&a.priority)
|
|
143
|
+
.then(b.max_width.cmp(&a.max_width))
|
|
144
|
+
.then(b.value_len.cmp(&a.value_len))
|
|
145
|
+
.then(a.name.cmp(&b.name))
|
|
127
146
|
});
|
|
128
147
|
|
|
129
148
|
let newline_types: HashSet<String> = terminals_sorted
|
|
@@ -158,15 +177,9 @@ impl BasicLexer {
|
|
|
158
177
|
let lark_lexer = py.import("lark.lexer")?;
|
|
159
178
|
let create_unless_fn = lark_lexer.getattr("_create_unless")?;
|
|
160
179
|
let call_chain_cls = lark_lexer.getattr("CallChain")?;
|
|
161
|
-
|
|
162
|
-
let
|
|
163
|
-
|
|
164
|
-
let py_terminals = pyo3::types::PyList::empty(py);
|
|
165
|
-
for t in &self.terminals {
|
|
166
|
-
let pattern_obj = pattern_re_cls.call1((&t.pattern,))?;
|
|
167
|
-
let td = terminal_def_cls.call1((&t.name, pattern_obj, t.priority))?;
|
|
168
|
-
py_terminals.append(td)?;
|
|
169
|
-
}
|
|
180
|
+
|
|
181
|
+
let py_terminals =
|
|
182
|
+
pyo3::types::PyList::new(py, self.terminals.iter().map(|t| t.py_def.bind(py)))?;
|
|
170
183
|
|
|
171
184
|
let result = create_unless_fn.call1((
|
|
172
185
|
py_terminals,
|
|
@@ -29,7 +29,7 @@ fn lark_rust(m: &Bound<'_, PyModule>) -> PyResult<()> {
|
|
|
29
29
|
plugins.set_item("BasicLexer", m.getattr("BasicLexer")?)?;
|
|
30
30
|
plugins.set_item("LexerThread", m.getattr("LexerThread")?)?;
|
|
31
31
|
m.add("plugins", plugins)?;
|
|
32
|
-
m.add("__version__", "0.
|
|
32
|
+
m.add("__version__", "0.2.2")?;
|
|
33
33
|
|
|
34
34
|
Ok(())
|
|
35
35
|
}
|
|
@@ -6,6 +6,9 @@ pub struct Scanner {
|
|
|
6
6
|
combined_re: Option<Regex>,
|
|
7
7
|
individual_patterns: Vec<(String, Regex)>,
|
|
8
8
|
group_to_terminal: Vec<String>,
|
|
9
|
+
/// Capture-group index of each terminal in `combined_re`. Terminal
|
|
10
|
+
/// patterns may contain their own groups, so indices are not contiguous.
|
|
11
|
+
group_indices: Vec<usize>,
|
|
9
12
|
}
|
|
10
13
|
|
|
11
14
|
impl Scanner {
|
|
@@ -27,16 +30,33 @@ impl Scanner {
|
|
|
27
30
|
if let Ok(re) = Regex::new(pattern) {
|
|
28
31
|
individual_patterns.push((name.clone(), re));
|
|
29
32
|
}
|
|
30
|
-
pattern_parts.push(format!("({})", pattern));
|
|
33
|
+
pattern_parts.push(format!("(?P<{}>{})", name, pattern));
|
|
31
34
|
group_to_terminal.push(name.clone());
|
|
32
35
|
}
|
|
33
36
|
|
|
34
37
|
let combined_re = Regex::new(&pattern_parts.join("|")).ok();
|
|
35
38
|
|
|
39
|
+
let group_indices = match combined_re {
|
|
40
|
+
Some(ref re) => {
|
|
41
|
+
let names: Vec<Option<&str>> = re.capture_names().collect();
|
|
42
|
+
group_to_terminal
|
|
43
|
+
.iter()
|
|
44
|
+
.map(|n| {
|
|
45
|
+
names
|
|
46
|
+
.iter()
|
|
47
|
+
.position(|g| *g == Some(n.as_str()))
|
|
48
|
+
.unwrap_or(0)
|
|
49
|
+
})
|
|
50
|
+
.collect()
|
|
51
|
+
}
|
|
52
|
+
None => Vec::new(),
|
|
53
|
+
};
|
|
54
|
+
|
|
36
55
|
Ok(Scanner {
|
|
37
56
|
combined_re,
|
|
38
57
|
individual_patterns,
|
|
39
58
|
group_to_terminal,
|
|
59
|
+
group_indices,
|
|
40
60
|
})
|
|
41
61
|
}
|
|
42
62
|
|
|
@@ -54,9 +74,9 @@ impl Scanner {
|
|
|
54
74
|
if full.start() == 0 {
|
|
55
75
|
let match_end = full.end();
|
|
56
76
|
let matched = &text[pos..pos + match_end];
|
|
57
|
-
// Find which group matched
|
|
58
|
-
for (
|
|
59
|
-
if caps.get(
|
|
77
|
+
// Find which terminal's group matched
|
|
78
|
+
for (name, &idx) in self.group_to_terminal.iter().zip(&self.group_indices) {
|
|
79
|
+
if caps.get(idx).is_some_and(|m| m.start() == 0) {
|
|
60
80
|
return Some((matched, name.clone()));
|
|
61
81
|
}
|
|
62
82
|
}
|
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
"""Regression tests for https://github.com/robertozimek/lark-rust/issues/1"""
|
|
2
|
+
import pytest
|
|
3
|
+
from lark import Lark
|
|
4
|
+
from lark.exceptions import UnexpectedCharacters, UnexpectedToken
|
|
5
|
+
import lark_rust
|
|
6
|
+
|
|
7
|
+
GRAMMAR = r"""
|
|
8
|
+
INT_NUMBER: /\d+/
|
|
9
|
+
FLOAT_NUMBER: /\d*\.\d+|\d+e(\+|\-)\d+/
|
|
10
|
+
VARIABLE: /[a-zA-Z_]\w*/
|
|
11
|
+
NEWLINE: "\n"
|
|
12
|
+
PLUS: "+"
|
|
13
|
+
EQ: "="
|
|
14
|
+
LPAR: "("
|
|
15
|
+
RPAR: ")"
|
|
16
|
+
LBRACK: "["
|
|
17
|
+
RBRACK: "]"
|
|
18
|
+
COMMA: ","
|
|
19
|
+
TYPE_COMPLEX: "complex"
|
|
20
|
+
|
|
21
|
+
%ignore /[ \t]+/
|
|
22
|
+
|
|
23
|
+
start: (NEWLINE* instruction)+ NEWLINE*
|
|
24
|
+
|
|
25
|
+
instruction: expr
|
|
26
|
+
| VARIABLE EQ expr
|
|
27
|
+
|
|
28
|
+
expr: expr PLUS atom
|
|
29
|
+
| atom
|
|
30
|
+
|
|
31
|
+
atom: INT_NUMBER
|
|
32
|
+
| FLOAT_NUMBER
|
|
33
|
+
| VARIABLE
|
|
34
|
+
| TYPE_COMPLEX LPAR expr COMMA expr RPAR
|
|
35
|
+
| LBRACK expr RBRACK
|
|
36
|
+
"""
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def _parsers():
|
|
40
|
+
kw = dict(parser="lalr", lexer="contextual")
|
|
41
|
+
return Lark(GRAMMAR, **kw), Lark(GRAMMAR, _plugins=lark_rust.plugins, **kw)
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def _tokens(tree):
|
|
45
|
+
return [(t.type, t.value) for t in tree.scan_values(lambda _: True)]
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
@pytest.mark.parametrize("text", [
|
|
49
|
+
"complex_number = complex(2,3)",
|
|
50
|
+
"complexity + complex(1, 2.5)",
|
|
51
|
+
"x = 1e+5 + [y]",
|
|
52
|
+
"a = 1\nb = a + 2\n",
|
|
53
|
+
])
|
|
54
|
+
def test_matches_lark(text):
|
|
55
|
+
ref, rust = _parsers()
|
|
56
|
+
assert _tokens(rust.parse(text)) == _tokens(ref.parse(text))
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def test_keyword_prefix_lexed_as_variable():
|
|
60
|
+
_, rust = _parsers()
|
|
61
|
+
tokens = _tokens(rust.parse("complex_number = complex(2,3)"))
|
|
62
|
+
assert tokens[0] == ("VARIABLE", "complex_number")
|
|
63
|
+
assert tokens[2] == ("TYPE_COMPLEX", "complex")
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def test_terminal_with_inner_group_does_not_shift_types():
|
|
67
|
+
# FLOAT_NUMBER contains a capture group; tokens after it must keep their types.
|
|
68
|
+
_, rust = _parsers()
|
|
69
|
+
tokens = _tokens(rust.parse("x = 1e+5 + [y]"))
|
|
70
|
+
assert ("FLOAT_NUMBER", "1e+5") in tokens
|
|
71
|
+
assert ("LBRACK", "[") in tokens
|
|
72
|
+
assert ("EQ", "=") in tokens
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def test_basic_lexer_exposes_terminals_by_name():
|
|
76
|
+
_, rust = _parsers()
|
|
77
|
+
root = rust.parser.lexer.root_lexer
|
|
78
|
+
assert "VARIABLE" in root.terminals_by_name
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def test_unexpected_token_from_root_lexer():
|
|
82
|
+
# "=" is valid for the root lexer but not after EQ, so the contextual
|
|
83
|
+
# lexer raises UnexpectedToken built from root_lexer.terminals_by_name.
|
|
84
|
+
ref, rust = _parsers()
|
|
85
|
+
with pytest.raises(UnexpectedToken) as ref_exc:
|
|
86
|
+
ref.parse("x = = 1")
|
|
87
|
+
with pytest.raises(UnexpectedToken) as rust_exc:
|
|
88
|
+
rust.parse("x = = 1")
|
|
89
|
+
assert rust_exc.value.token.type == ref_exc.value.token.type == "EQ"
|
|
90
|
+
assert rust_exc.value.expected == ref_exc.value.expected
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def test_unexpected_characters():
|
|
94
|
+
_, rust = _parsers()
|
|
95
|
+
with pytest.raises(UnexpectedCharacters):
|
|
96
|
+
rust.parse("x = 1 $ 2")
|
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
"""Exact reproduction from https://github.com/robertozimek/lark-rust/issues/1"""
|
|
2
|
+
from lark import Lark
|
|
3
|
+
import lark_rust
|
|
4
|
+
|
|
5
|
+
GRAMMAR = r"""// ==================== TOKENS ====================
|
|
6
|
+
INT_NUMBER: /\d+/
|
|
7
|
+
FLOAT_NUMBER: /\d*\.\d+|\d+e(\+|\-)\d+/
|
|
8
|
+
VARIABLE: /[a-zA-Z_]\w*/
|
|
9
|
+
NEWLINE: "\n"
|
|
10
|
+
PLUS: "+"
|
|
11
|
+
MINUS: "-"
|
|
12
|
+
MUL: "*"
|
|
13
|
+
DIV: "/"
|
|
14
|
+
MOD: "%"
|
|
15
|
+
POWER: "**"
|
|
16
|
+
EQ: "="
|
|
17
|
+
LPAR: "("
|
|
18
|
+
RPAR: ")"
|
|
19
|
+
LBRACK: "["
|
|
20
|
+
RBRACK: "]"
|
|
21
|
+
COMMA: ","
|
|
22
|
+
COLON: ":"
|
|
23
|
+
TYPE_COMPLEX: "complex"
|
|
24
|
+
TYPE_FLOAT: "float"
|
|
25
|
+
TYPE_INT: "int"
|
|
26
|
+
TYPE_VECTOR: "vector"
|
|
27
|
+
SUM_KEYWORD: "sum"
|
|
28
|
+
MEAN_KEYWORD: "mean"
|
|
29
|
+
DOT_KEYWORD: "dot"
|
|
30
|
+
PRINT_KEYWORD: "print"
|
|
31
|
+
|
|
32
|
+
%ignore /[ \t]+/
|
|
33
|
+
%ignore /\/\/.*\n/
|
|
34
|
+
|
|
35
|
+
start: vec_lang_program
|
|
36
|
+
|
|
37
|
+
vec_lang_program: vec_lang_instructions_sequence
|
|
38
|
+
|
|
39
|
+
vec_lang_instructions_sequence: (NEWLINE* vec_lang_instruction)+ NEWLINE*
|
|
40
|
+
|
|
41
|
+
vec_lang_instruction: arithmetic_expression_level_1
|
|
42
|
+
| function_decl
|
|
43
|
+
| variable_expression EQ arithmetic_expression_level_1
|
|
44
|
+
| PRINT_KEYWORD LPAR function_args RPAR
|
|
45
|
+
|
|
46
|
+
arithmetic_expression_level_1: arithmetic_expression_level_1 PLUS arithmetic_expression_level_2
|
|
47
|
+
| arithmetic_expression_level_1 MINUS arithmetic_expression_level_2
|
|
48
|
+
| arithmetic_expression_level_2
|
|
49
|
+
|
|
50
|
+
arithmetic_expression_level_2: arithmetic_expression_level_2 MUL arithmetic_expression_level_3
|
|
51
|
+
| arithmetic_expression_level_2 DIV arithmetic_expression_level_3
|
|
52
|
+
| arithmetic_expression_level_2 MOD arithmetic_expression_level_3
|
|
53
|
+
| arithmetic_expression_level_3
|
|
54
|
+
|
|
55
|
+
arithmetic_expression_level_3: arithmetic_expression_level_3 POWER arithmetic_expression_level_4
|
|
56
|
+
| arithmetic_expression_level_4
|
|
57
|
+
|
|
58
|
+
arithmetic_expression_level_4: number_expression
|
|
59
|
+
| variable_expression
|
|
60
|
+
| vector
|
|
61
|
+
| indexing
|
|
62
|
+
| function_call
|
|
63
|
+
| LPAR arithmetic_expression_level_1 RPAR
|
|
64
|
+
|
|
65
|
+
number_expression: number
|
|
66
|
+
| complex_number
|
|
67
|
+
|
|
68
|
+
number: INT_NUMBER
|
|
69
|
+
| FLOAT_NUMBER
|
|
70
|
+
| PLUS INT_NUMBER
|
|
71
|
+
| MINUS INT_NUMBER
|
|
72
|
+
| PLUS FLOAT_NUMBER
|
|
73
|
+
| MINUS FLOAT_NUMBER
|
|
74
|
+
|
|
75
|
+
complex_number: TYPE_COMPLEX LPAR number COMMA number RPAR
|
|
76
|
+
| number VARIABLE
|
|
77
|
+
|
|
78
|
+
variable_expression: VARIABLE
|
|
79
|
+
|
|
80
|
+
vector: LBRACK components RBRACK
|
|
81
|
+
| LBRACK range RBRACK
|
|
82
|
+
| slicing
|
|
83
|
+
|
|
84
|
+
components: arithmetic_expression_level_1
|
|
85
|
+
| components COMMA arithmetic_expression_level_1
|
|
86
|
+
|
|
87
|
+
range: INT_NUMBER COLON INT_NUMBER
|
|
88
|
+
| MINUS INT_NUMBER COLON INT_NUMBER
|
|
89
|
+
| INT_NUMBER COLON MINUS INT_NUMBER
|
|
90
|
+
| MINUS INT_NUMBER COLON MINUS INT_NUMBER
|
|
91
|
+
|
|
92
|
+
indexing: variable_expression LBRACK INT_NUMBER RBRACK
|
|
93
|
+
| vector LBRACK INT_NUMBER RBRACK
|
|
94
|
+
|
|
95
|
+
slicing: variable_expression LBRACK range RBRACK
|
|
96
|
+
| vector LBRACK range RBRACK
|
|
97
|
+
|
|
98
|
+
function_call: variable_expression LPAR function_args RPAR
|
|
99
|
+
| SUM_KEYWORD LPAR function_args RPAR
|
|
100
|
+
| MEAN_KEYWORD LPAR function_args RPAR
|
|
101
|
+
| DOT_KEYWORD LPAR function_args RPAR
|
|
102
|
+
|
|
103
|
+
function_args: arithmetic_expression_level_1
|
|
104
|
+
| function_args COMMA arithmetic_expression_level_1
|
|
105
|
+
|
|
106
|
+
function_decl: variable_expression LPAR function_decl_args RPAR EQ arithmetic_expression_level_1
|
|
107
|
+
|
|
108
|
+
function_decl_args: variable_expression COLON type
|
|
109
|
+
| function_decl_args COMMA variable_expression COLON type
|
|
110
|
+
|
|
111
|
+
type: TYPE_COMPLEX
|
|
112
|
+
| TYPE_FLOAT
|
|
113
|
+
| TYPE_INT
|
|
114
|
+
| TYPE_VECTOR
|
|
115
|
+
"""
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
def test_issue_1_exact_repro():
|
|
119
|
+
text = 'complex_number = complex(2,3)'
|
|
120
|
+
parser = Lark(GRAMMAR, parser='lalr', lexer='contextual', _plugins=lark_rust.plugins)
|
|
121
|
+
tree = parser.parse(text)
|
|
122
|
+
|
|
123
|
+
expected = Lark(GRAMMAR, parser='lalr', lexer='contextual').parse(text)
|
|
124
|
+
assert tree == expected
|
|
@@ -1,76 +0,0 @@
|
|
|
1
|
-
name: Release
|
|
2
|
-
|
|
3
|
-
on:
|
|
4
|
-
push:
|
|
5
|
-
tags:
|
|
6
|
-
- "v*"
|
|
7
|
-
|
|
8
|
-
permissions:
|
|
9
|
-
contents: read
|
|
10
|
-
|
|
11
|
-
jobs:
|
|
12
|
-
build-sdist:
|
|
13
|
-
runs-on: ubuntu-latest
|
|
14
|
-
steps:
|
|
15
|
-
- uses: actions/checkout@v4
|
|
16
|
-
- name: Build sdist
|
|
17
|
-
uses: PyO3/maturin-action@v1
|
|
18
|
-
with:
|
|
19
|
-
command: sdist
|
|
20
|
-
args: --out dist
|
|
21
|
-
- uses: actions/upload-artifact@v4
|
|
22
|
-
with:
|
|
23
|
-
name: sdist
|
|
24
|
-
path: dist/*.tar.gz
|
|
25
|
-
|
|
26
|
-
build-wheels:
|
|
27
|
-
runs-on: ${{ matrix.os }}
|
|
28
|
-
strategy:
|
|
29
|
-
fail-fast: false
|
|
30
|
-
matrix:
|
|
31
|
-
include:
|
|
32
|
-
# Linux x86_64
|
|
33
|
-
- os: ubuntu-latest
|
|
34
|
-
target: x86_64
|
|
35
|
-
# Linux aarch64
|
|
36
|
-
- os: ubuntu-latest
|
|
37
|
-
target: aarch64
|
|
38
|
-
# macOS universal2 (x86_64 + arm64 fat binary)
|
|
39
|
-
- os: macos-latest
|
|
40
|
-
target: universal2-apple-darwin
|
|
41
|
-
# Windows x86_64
|
|
42
|
-
- os: windows-latest
|
|
43
|
-
target: x64
|
|
44
|
-
steps:
|
|
45
|
-
- uses: actions/checkout@v4
|
|
46
|
-
- uses: actions/setup-python@v5
|
|
47
|
-
with:
|
|
48
|
-
python-version: "3.x"
|
|
49
|
-
- name: Build wheels
|
|
50
|
-
uses: PyO3/maturin-action@v1
|
|
51
|
-
with:
|
|
52
|
-
target: ${{ matrix.target }}
|
|
53
|
-
args: --release --out dist --find-interpreter
|
|
54
|
-
sccache: "true"
|
|
55
|
-
manylinux: auto
|
|
56
|
-
- uses: actions/upload-artifact@v4
|
|
57
|
-
with:
|
|
58
|
-
name: wheels-${{ matrix.os }}-${{ matrix.target }}
|
|
59
|
-
path: dist/*.whl
|
|
60
|
-
|
|
61
|
-
publish:
|
|
62
|
-
needs: [build-sdist, build-wheels]
|
|
63
|
-
runs-on: ubuntu-latest
|
|
64
|
-
environment: pypi
|
|
65
|
-
permissions:
|
|
66
|
-
id-token: write
|
|
67
|
-
steps:
|
|
68
|
-
- uses: actions/download-artifact@v4
|
|
69
|
-
with:
|
|
70
|
-
path: dist
|
|
71
|
-
merge-multiple: true
|
|
72
|
-
- name: Publish to PyPI
|
|
73
|
-
uses: PyO3/maturin-action@v1
|
|
74
|
-
with:
|
|
75
|
-
command: upload
|
|
76
|
-
args: --non-interactive --skip-existing dist/*
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|