lark-rust 0.2.0__tar.gz → 0.2.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {lark_rust-0.2.0 → lark_rust-0.2.2}/.github/workflows/ci.yml +18 -1
- lark_rust-0.2.2/.github/workflows/release.yml +85 -0
- {lark_rust-0.2.0 → lark_rust-0.2.2}/Cargo.lock +29 -29
- {lark_rust-0.2.0 → lark_rust-0.2.2}/Cargo.toml +1 -1
- {lark_rust-0.2.0 → lark_rust-0.2.2}/PKG-INFO +4 -4
- {lark_rust-0.2.0 → lark_rust-0.2.2}/README.md +3 -3
- {lark_rust-0.2.0 → lark_rust-0.2.2}/pyproject.toml +1 -1
- {lark_rust-0.2.0 → lark_rust-0.2.2}/python/lark_rust/__init__.py +14 -6
- {lark_rust-0.2.0 → lark_rust-0.2.2}/src/lexer.rs +25 -12
- {lark_rust-0.2.0 → lark_rust-0.2.2}/src/lib.rs +1 -1
- {lark_rust-0.2.0 → lark_rust-0.2.2}/src/parser.rs +109 -1
- {lark_rust-0.2.0 → lark_rust-0.2.2}/src/scanner.rs +24 -4
- lark_rust-0.2.2/tests/test_contextual_lexer.py +96 -0
- lark_rust-0.2.2/tests/test_issue_1.py +124 -0
- lark_rust-0.2.0/.github/workflows/release.yml +0 -76
- {lark_rust-0.2.0 → lark_rust-0.2.2}/.gitignore +0 -0
- {lark_rust-0.2.0 → lark_rust-0.2.2}/LICENSE +0 -0
- {lark_rust-0.2.0 → lark_rust-0.2.2}/benchmarks/bench.py +0 -0
- {lark_rust-0.2.0 → lark_rust-0.2.2}/src/lexer_state.rs +0 -0
- {lark_rust-0.2.0 → lark_rust-0.2.2}/src/lexer_thread.rs +0 -0
- {lark_rust-0.2.0 → lark_rust-0.2.2}/src/line_counter.rs +0 -0
- {lark_rust-0.2.0 → lark_rust-0.2.2}/src/token.rs +0 -0
- {lark_rust-0.2.0 → lark_rust-0.2.2}/tests/test_cython_compat.py +0 -0
- {lark_rust-0.2.0 → lark_rust-0.2.2}/tests/test_integration.py +0 -0
- {lark_rust-0.2.0 → lark_rust-0.2.2}/tests/test_lexer.py +0 -0
- {lark_rust-0.2.0 → lark_rust-0.2.2}/tests/test_lexer_state.py +0 -0
- {lark_rust-0.2.0 → lark_rust-0.2.2}/tests/test_lexer_thread.py +0 -0
- {lark_rust-0.2.0 → lark_rust-0.2.2}/tests/test_line_counter.py +0 -0
- {lark_rust-0.2.0 → lark_rust-0.2.2}/tests/test_scanner.py +0 -0
- {lark_rust-0.2.0 → lark_rust-0.2.2}/tests/test_token.py +0 -0
|
@@ -14,6 +14,13 @@ jobs:
|
|
|
14
14
|
matrix:
|
|
15
15
|
os: [ubuntu-latest, macos-latest, windows-latest]
|
|
16
16
|
python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"]
|
|
17
|
+
exclude:
|
|
18
|
+
# Python 3.9 isn't available on ubuntu-latest (26.04)
|
|
19
|
+
- os: ubuntu-latest
|
|
20
|
+
python-version: "3.9"
|
|
21
|
+
include:
|
|
22
|
+
- os: ubuntu-24.04
|
|
23
|
+
python-version: "3.9"
|
|
17
24
|
steps:
|
|
18
25
|
- uses: actions/checkout@v4
|
|
19
26
|
- uses: actions/setup-python@v5
|
|
@@ -21,11 +28,20 @@ jobs:
|
|
|
21
28
|
python-version: ${{ matrix.python-version }}
|
|
22
29
|
- name: Install Rust
|
|
23
30
|
uses: dtolnay/rust-toolchain@stable
|
|
31
|
+
- uses: Swatinem/rust-cache@v2
|
|
24
32
|
- name: Build and install
|
|
33
|
+
# Full LTO is only needed for release wheels; skip it to speed up CI.
|
|
34
|
+
env:
|
|
35
|
+
CARGO_PROFILE_RELEASE_LTO: "false"
|
|
36
|
+
CARGO_PROFILE_RELEASE_CODEGEN_UNITS: "16"
|
|
37
|
+
# bash with -e so a failed build stops here on Windows too, and
|
|
38
|
+
# --no-index so we never fall back to the PyPI release.
|
|
39
|
+
shell: bash
|
|
25
40
|
run: |
|
|
41
|
+
set -e
|
|
26
42
|
pip install maturin lark pytest
|
|
27
43
|
maturin build --release
|
|
28
|
-
pip install --find-links target/wheels lark-rust
|
|
44
|
+
pip install --no-index --find-links target/wheels lark-rust
|
|
29
45
|
- name: Run tests
|
|
30
46
|
run: pytest tests/ -v
|
|
31
47
|
|
|
@@ -37,6 +53,7 @@ jobs:
|
|
|
37
53
|
uses: dtolnay/rust-toolchain@stable
|
|
38
54
|
with:
|
|
39
55
|
components: clippy, rustfmt
|
|
56
|
+
- uses: Swatinem/rust-cache@v2
|
|
40
57
|
- name: Cargo clippy
|
|
41
58
|
run: cargo clippy --all-targets -- -D warnings
|
|
42
59
|
- name: Cargo fmt check
|
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
name: Release
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
tags:
|
|
6
|
+
- "v*"
|
|
7
|
+
workflow_dispatch:
|
|
8
|
+
|
|
9
|
+
permissions:
|
|
10
|
+
contents: read
|
|
11
|
+
|
|
12
|
+
jobs:
|
|
13
|
+
build-sdist:
|
|
14
|
+
runs-on: ubuntu-latest
|
|
15
|
+
steps:
|
|
16
|
+
- uses: actions/checkout@v4
|
|
17
|
+
- name: Build sdist
|
|
18
|
+
uses: PyO3/maturin-action@v1
|
|
19
|
+
with:
|
|
20
|
+
command: sdist
|
|
21
|
+
args: --out dist
|
|
22
|
+
- uses: actions/upload-artifact@v4
|
|
23
|
+
with:
|
|
24
|
+
name: sdist
|
|
25
|
+
path: dist/*.tar.gz
|
|
26
|
+
|
|
27
|
+
build-wheels:
|
|
28
|
+
name: build-wheels (${{ matrix.platform.target }}, ${{ matrix.label || matrix.python }})
|
|
29
|
+
runs-on: ${{ matrix.platform.os }}
|
|
30
|
+
strategy:
|
|
31
|
+
fail-fast: false
|
|
32
|
+
# One interpreter per job so builds run in parallel.
|
|
33
|
+
# Interpreters are listed explicitly: pyo3 0.25 supports up to 3.14,
|
|
34
|
+
# so --find-interpreter fails once runners ship a newer Python.
|
|
35
|
+
matrix:
|
|
36
|
+
platform:
|
|
37
|
+
- { os: ubuntu-latest, target: x86_64 }
|
|
38
|
+
- { os: ubuntu-latest, target: aarch64 }
|
|
39
|
+
# macOS universal2 (x86_64 + arm64 fat binary)
|
|
40
|
+
- { os: macos-latest, target: universal2-apple-darwin }
|
|
41
|
+
- { os: windows-latest, target: x64 }
|
|
42
|
+
python: ["3.10", "3.11", "3.12", "3.13", "3.14"]
|
|
43
|
+
include:
|
|
44
|
+
# Extra Linux-only interpreters
|
|
45
|
+
- platform: { os: ubuntu-latest, target: x86_64 }
|
|
46
|
+
python: 3.8 3.9 3.14t pypy3.9 pypy3.10 pypy3.11
|
|
47
|
+
label: extras
|
|
48
|
+
- platform: { os: ubuntu-latest, target: aarch64 }
|
|
49
|
+
python: 3.8 3.9 3.14t pypy3.9 pypy3.10 pypy3.11
|
|
50
|
+
label: extras
|
|
51
|
+
steps:
|
|
52
|
+
- uses: actions/checkout@v4
|
|
53
|
+
- uses: actions/setup-python@v5
|
|
54
|
+
if: runner.os != 'Linux'
|
|
55
|
+
with:
|
|
56
|
+
python-version: ${{ matrix.python }}
|
|
57
|
+
- name: Build wheels
|
|
58
|
+
uses: PyO3/maturin-action@v1
|
|
59
|
+
with:
|
|
60
|
+
target: ${{ matrix.platform.target }}
|
|
61
|
+
args: --release --out dist --interpreter ${{ matrix.python }}
|
|
62
|
+
sccache: "true"
|
|
63
|
+
manylinux: auto
|
|
64
|
+
- uses: actions/upload-artifact@v4
|
|
65
|
+
with:
|
|
66
|
+
name: wheels-${{ matrix.platform.os }}-${{ matrix.platform.target }}-${{ matrix.label || matrix.python }}
|
|
67
|
+
path: dist/*.whl
|
|
68
|
+
|
|
69
|
+
publish:
|
|
70
|
+
needs: [build-sdist, build-wheels]
|
|
71
|
+
if: startsWith(github.ref, 'refs/tags/')
|
|
72
|
+
runs-on: ubuntu-latest
|
|
73
|
+
environment: pypi
|
|
74
|
+
permissions:
|
|
75
|
+
id-token: write
|
|
76
|
+
steps:
|
|
77
|
+
- uses: actions/download-artifact@v4
|
|
78
|
+
with:
|
|
79
|
+
path: dist
|
|
80
|
+
merge-multiple: true
|
|
81
|
+
- name: Publish to PyPI
|
|
82
|
+
uses: PyO3/maturin-action@v1
|
|
83
|
+
with:
|
|
84
|
+
command: upload
|
|
85
|
+
args: --non-interactive --skip-existing dist/*
|
|
@@ -4,18 +4,18 @@ version = 4
|
|
|
4
4
|
|
|
5
5
|
[[package]]
|
|
6
6
|
name = "aho-corasick"
|
|
7
|
-
version = "1.1.
|
|
7
|
+
version = "1.1.5"
|
|
8
8
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
9
|
-
checksum = "
|
|
9
|
+
checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba"
|
|
10
10
|
dependencies = [
|
|
11
11
|
"memchr",
|
|
12
12
|
]
|
|
13
13
|
|
|
14
14
|
[[package]]
|
|
15
15
|
name = "autocfg"
|
|
16
|
-
version = "1.5.
|
|
16
|
+
version = "1.5.1"
|
|
17
17
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
18
|
-
checksum = "
|
|
18
|
+
checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53"
|
|
19
19
|
|
|
20
20
|
[[package]]
|
|
21
21
|
name = "bit-set"
|
|
@@ -51,16 +51,16 @@ checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"
|
|
|
51
51
|
|
|
52
52
|
[[package]]
|
|
53
53
|
name = "indoc"
|
|
54
|
-
version = "2.0.
|
|
54
|
+
version = "2.0.8"
|
|
55
55
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
56
|
-
checksum = "
|
|
56
|
+
checksum = "a37b2691796cffeb8a8cd305ac66e65841559f147f4e63231d0eafa4db5384d1"
|
|
57
57
|
dependencies = [
|
|
58
58
|
"rustversion",
|
|
59
59
|
]
|
|
60
60
|
|
|
61
61
|
[[package]]
|
|
62
62
|
name = "lark-rust"
|
|
63
|
-
version = "0.2.
|
|
63
|
+
version = "0.2.2"
|
|
64
64
|
dependencies = [
|
|
65
65
|
"fancy-regex",
|
|
66
66
|
"pyo3",
|
|
@@ -68,15 +68,15 @@ dependencies = [
|
|
|
68
68
|
|
|
69
69
|
[[package]]
|
|
70
70
|
name = "libc"
|
|
71
|
-
version = "0.2.
|
|
71
|
+
version = "0.2.191"
|
|
72
72
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
73
|
-
checksum = "
|
|
73
|
+
checksum = "65cd87c9a6d60536580b316ad985726104730a5151c99845f2490428ef91240a"
|
|
74
74
|
|
|
75
75
|
[[package]]
|
|
76
76
|
name = "memchr"
|
|
77
|
-
version = "2.8.
|
|
77
|
+
version = "2.8.3"
|
|
78
78
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
79
|
-
checksum = "
|
|
79
|
+
checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98"
|
|
80
80
|
|
|
81
81
|
[[package]]
|
|
82
82
|
name = "memoffset"
|
|
@@ -95,15 +95,15 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50"
|
|
|
95
95
|
|
|
96
96
|
[[package]]
|
|
97
97
|
name = "portable-atomic"
|
|
98
|
-
version = "1.
|
|
98
|
+
version = "1.15.0"
|
|
99
99
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
100
|
-
checksum = "
|
|
100
|
+
checksum = "05c8b63e8d9609db387f0324918f81d68fe27748f084ef092fb35954d0539a85"
|
|
101
101
|
|
|
102
102
|
[[package]]
|
|
103
103
|
name = "proc-macro2"
|
|
104
|
-
version = "1.0.
|
|
104
|
+
version = "1.0.107"
|
|
105
105
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
106
|
-
checksum = "
|
|
106
|
+
checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9"
|
|
107
107
|
dependencies = [
|
|
108
108
|
"unicode-ident",
|
|
109
109
|
]
|
|
@@ -172,18 +172,18 @@ dependencies = [
|
|
|
172
172
|
|
|
173
173
|
[[package]]
|
|
174
174
|
name = "quote"
|
|
175
|
-
version = "1.0.
|
|
175
|
+
version = "1.0.47"
|
|
176
176
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
177
|
-
checksum = "
|
|
177
|
+
checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001"
|
|
178
178
|
dependencies = [
|
|
179
179
|
"proc-macro2",
|
|
180
180
|
]
|
|
181
181
|
|
|
182
182
|
[[package]]
|
|
183
183
|
name = "regex-automata"
|
|
184
|
-
version = "0.4.
|
|
184
|
+
version = "0.4.19"
|
|
185
185
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
186
|
-
checksum = "
|
|
186
|
+
checksum = "41d60ef07df457506a0a1a2073112eb4aae5733dbd2813c3da77d1b0fd441158"
|
|
187
187
|
dependencies = [
|
|
188
188
|
"aho-corasick",
|
|
189
189
|
"memchr",
|
|
@@ -192,21 +192,21 @@ dependencies = [
|
|
|
192
192
|
|
|
193
193
|
[[package]]
|
|
194
194
|
name = "regex-syntax"
|
|
195
|
-
version = "0.8.
|
|
195
|
+
version = "0.8.12"
|
|
196
196
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
197
|
-
checksum = "
|
|
197
|
+
checksum = "d902046e6a28a1e4c3cab97258a0ce11e994903f01c2d97a9447a0b3df85f4f6"
|
|
198
198
|
|
|
199
199
|
[[package]]
|
|
200
200
|
name = "rustversion"
|
|
201
|
-
version = "1.0.
|
|
201
|
+
version = "1.0.23"
|
|
202
202
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
203
|
-
checksum = "
|
|
203
|
+
checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f"
|
|
204
204
|
|
|
205
205
|
[[package]]
|
|
206
206
|
name = "syn"
|
|
207
|
-
version = "2.0.
|
|
207
|
+
version = "2.0.119"
|
|
208
208
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
209
|
-
checksum = "
|
|
209
|
+
checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297"
|
|
210
210
|
dependencies = [
|
|
211
211
|
"proc-macro2",
|
|
212
212
|
"quote",
|
|
@@ -215,15 +215,15 @@ dependencies = [
|
|
|
215
215
|
|
|
216
216
|
[[package]]
|
|
217
217
|
name = "target-lexicon"
|
|
218
|
-
version = "0.13.
|
|
218
|
+
version = "0.13.6"
|
|
219
219
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
220
|
-
checksum = "
|
|
220
|
+
checksum = "8c447be3a2abfa84b2d91680c56600bd6b2873866de27b36ec911be705e291da"
|
|
221
221
|
|
|
222
222
|
[[package]]
|
|
223
223
|
name = "unicode-ident"
|
|
224
|
-
version = "1.0.
|
|
224
|
+
version = "1.0.27"
|
|
225
225
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
226
|
-
checksum = "
|
|
226
|
+
checksum = "a2c754d6c33795a1c324727428e5a7dedb5b06195f9890bdbcba760d3e246563"
|
|
227
227
|
|
|
228
228
|
[[package]]
|
|
229
229
|
name = "unindent"
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: lark-rust
|
|
3
|
-
Version: 0.2.
|
|
3
|
+
Version: 0.2.2
|
|
4
4
|
Classifier: Development Status :: 4 - Beta
|
|
5
5
|
Classifier: Intended Audience :: Developers
|
|
6
6
|
Classifier: License :: OSI Approved :: MIT License
|
|
@@ -56,9 +56,9 @@ Measured on Apple M4 Pro, Python 3.13, median of 20 iterations:
|
|
|
56
56
|
|
|
57
57
|
| Workload | lark | lark-cython | lark-rust | vs lark | vs cython |
|
|
58
58
|
|---|---|---|---|---|---|
|
|
59
|
-
| JSON (3 KB nested) | 4.
|
|
60
|
-
| Arithmetic (deep) | 3.
|
|
61
|
-
| Lexer (10k words) | 28.
|
|
59
|
+
| JSON (3 KB nested) | 4.16 ms | 2.45 ms | **1.73 ms** | 2.4x | 1.4x |
|
|
60
|
+
| Arithmetic (deep) | 3.08 ms | 2.00 ms | **1.18 ms** | 2.6x | 1.7x |
|
|
61
|
+
| Lexer (10k words) | 28.41 ms | 16.89 ms | **10.61 ms** | 2.7x | 1.6x |
|
|
62
62
|
|
|
63
63
|
Reproduce with `python benchmarks/bench.py` (install `lark-cython` for the three-way comparison).
|
|
64
64
|
|
|
@@ -26,9 +26,9 @@ Measured on Apple M4 Pro, Python 3.13, median of 20 iterations:
|
|
|
26
26
|
|
|
27
27
|
| Workload | lark | lark-cython | lark-rust | vs lark | vs cython |
|
|
28
28
|
|---|---|---|---|---|---|
|
|
29
|
-
| JSON (3 KB nested) | 4.
|
|
30
|
-
| Arithmetic (deep) | 3.
|
|
31
|
-
| Lexer (10k words) | 28.
|
|
29
|
+
| JSON (3 KB nested) | 4.16 ms | 2.45 ms | **1.73 ms** | 2.4x | 1.4x |
|
|
30
|
+
| Arithmetic (deep) | 3.08 ms | 2.00 ms | **1.18 ms** | 2.6x | 1.7x |
|
|
31
|
+
| Lexer (10k words) | 28.41 ms | 16.89 ms | **10.61 ms** | 2.7x | 1.6x |
|
|
32
32
|
|
|
33
33
|
Reproduce with `python benchmarks/bench.py` (install `lark-cython` for the three-way comparison).
|
|
34
34
|
|
|
@@ -191,13 +191,23 @@ class _Parser:
|
|
|
191
191
|
|
|
192
192
|
def parse_from_state(self, state, last_token=None):
|
|
193
193
|
try:
|
|
194
|
-
token = last_token
|
|
195
194
|
lexer = state.lexer
|
|
195
|
+
compiled = state.parse_conf._compiled
|
|
196
|
+
|
|
197
|
+
if (last_token is None
|
|
198
|
+
and hasattr(lexer, 'state')
|
|
199
|
+
and isinstance(lexer.lexer, BasicLexer)):
|
|
200
|
+
# Fast path: LexerThread with BasicLexer — run entire
|
|
201
|
+
# lex+parse loop in a single Rust call.
|
|
202
|
+
return compiled.parse_loop(
|
|
203
|
+
lexer.lexer, lexer.state,
|
|
204
|
+
state.state_stack, state.value_stack,
|
|
205
|
+
state,
|
|
206
|
+
)
|
|
196
207
|
|
|
197
|
-
#
|
|
198
|
-
|
|
208
|
+
# Fallback: ContextualLexer or resume with last_token
|
|
209
|
+
token = last_token
|
|
199
210
|
if hasattr(lexer, 'state'):
|
|
200
|
-
# LexerThread path
|
|
201
211
|
inner_lexer = lexer.lexer
|
|
202
212
|
lexer_state = lexer.state
|
|
203
213
|
try:
|
|
@@ -207,8 +217,6 @@ class _Parser:
|
|
|
207
217
|
except EOFError:
|
|
208
218
|
pass
|
|
209
219
|
else:
|
|
210
|
-
# ContextualLexer path — uses lex() which needs
|
|
211
|
-
# lexer_state + parser_state together
|
|
212
220
|
for token in lexer.lex(state):
|
|
213
221
|
assert token is not None
|
|
214
222
|
state.feed_token(token)
|
|
@@ -10,6 +10,12 @@ pub struct TerminalDef {
|
|
|
10
10
|
pub name: String,
|
|
11
11
|
pub pattern: String,
|
|
12
12
|
pub priority: i32,
|
|
13
|
+
/// Sort keys matching lark's `BasicLexer` ordering.
|
|
14
|
+
pub max_width: u64,
|
|
15
|
+
pub value_len: usize,
|
|
16
|
+
/// The original lark `TerminalDef`, passed to `_create_unless` so that
|
|
17
|
+
/// `PatternStr`/`PatternRE` distinctions and flags are preserved.
|
|
18
|
+
pub py_def: Py<PyAny>,
|
|
13
19
|
}
|
|
14
20
|
|
|
15
21
|
fn pattern_has_newline(pattern: &str) -> bool {
|
|
@@ -39,10 +45,19 @@ fn extract_terminals(conf: &Bound<'_, PyAny>, py: Python<'_>) -> PyResult<Vec<Te
|
|
|
39
45
|
let regex_result: Py<PyAny> = to_regexp_fn.call0()?.into();
|
|
40
46
|
let pattern_str: String = regex_result.extract(py)?;
|
|
41
47
|
let priority: i32 = t.getattr("priority")?.extract()?;
|
|
48
|
+
// max_width may be sre's MAXREPEAT; saturate rather than fail.
|
|
49
|
+
let max_width: u64 = pattern_obj
|
|
50
|
+
.getattr("max_width")?
|
|
51
|
+
.extract()
|
|
52
|
+
.unwrap_or(u64::MAX);
|
|
53
|
+
let value_len: usize = pattern_obj.getattr("value")?.len()?;
|
|
42
54
|
terminals.push(TerminalDef {
|
|
43
55
|
name,
|
|
44
56
|
pattern: pattern_str,
|
|
45
57
|
priority,
|
|
58
|
+
max_width,
|
|
59
|
+
value_len,
|
|
60
|
+
py_def: t.unbind(),
|
|
46
61
|
});
|
|
47
62
|
}
|
|
48
63
|
Ok(terminals)
|
|
@@ -78,6 +93,7 @@ pub struct BasicLexer {
|
|
|
78
93
|
re_module: Py<PyAny>,
|
|
79
94
|
g_regex_flags: i32,
|
|
80
95
|
use_bytes: bool,
|
|
96
|
+
#[pyo3(get)]
|
|
81
97
|
terminals_by_name: Py<PyAny>,
|
|
82
98
|
}
|
|
83
99
|
|
|
@@ -120,10 +136,13 @@ impl BasicLexer {
|
|
|
120
136
|
}
|
|
121
137
|
|
|
122
138
|
let mut terminals_sorted = terminals;
|
|
139
|
+
// Same order as lark: (-priority, -max_width, -len(value), name)
|
|
123
140
|
terminals_sorted.sort_by(|a, b| {
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
141
|
+
b.priority
|
|
142
|
+
.cmp(&a.priority)
|
|
143
|
+
.then(b.max_width.cmp(&a.max_width))
|
|
144
|
+
.then(b.value_len.cmp(&a.value_len))
|
|
145
|
+
.then(a.name.cmp(&b.name))
|
|
127
146
|
});
|
|
128
147
|
|
|
129
148
|
let newline_types: HashSet<String> = terminals_sorted
|
|
@@ -158,15 +177,9 @@ impl BasicLexer {
|
|
|
158
177
|
let lark_lexer = py.import("lark.lexer")?;
|
|
159
178
|
let create_unless_fn = lark_lexer.getattr("_create_unless")?;
|
|
160
179
|
let call_chain_cls = lark_lexer.getattr("CallChain")?;
|
|
161
|
-
|
|
162
|
-
let
|
|
163
|
-
|
|
164
|
-
let py_terminals = pyo3::types::PyList::empty(py);
|
|
165
|
-
for t in &self.terminals {
|
|
166
|
-
let pattern_obj = pattern_re_cls.call1((&t.pattern,))?;
|
|
167
|
-
let td = terminal_def_cls.call1((&t.name, pattern_obj, t.priority))?;
|
|
168
|
-
py_terminals.append(td)?;
|
|
169
|
-
}
|
|
180
|
+
|
|
181
|
+
let py_terminals =
|
|
182
|
+
pyo3::types::PyList::new(py, self.terminals.iter().map(|t| t.py_def.bind(py)))?;
|
|
170
183
|
|
|
171
184
|
let result = create_unless_fn.call1((
|
|
172
185
|
py_terminals,
|
|
@@ -29,7 +29,7 @@ fn lark_rust(m: &Bound<'_, PyModule>) -> PyResult<()> {
|
|
|
29
29
|
plugins.set_item("BasicLexer", m.getattr("BasicLexer")?)?;
|
|
30
30
|
plugins.set_item("LexerThread", m.getattr("LexerThread")?)?;
|
|
31
31
|
m.add("plugins", plugins)?;
|
|
32
|
-
m.add("__version__", "0.
|
|
32
|
+
m.add("__version__", "0.2.2")?;
|
|
33
33
|
|
|
34
34
|
Ok(())
|
|
35
35
|
}
|
|
@@ -2,6 +2,10 @@ use pyo3::prelude::*;
|
|
|
2
2
|
use pyo3::types::{PyDict, PyList, PySet, PyTuple};
|
|
3
3
|
use std::collections::HashMap;
|
|
4
4
|
|
|
5
|
+
use crate::lexer::BasicLexer;
|
|
6
|
+
use crate::lexer_state::LexerState;
|
|
7
|
+
use crate::token::Token;
|
|
8
|
+
|
|
5
9
|
/// A pre-compiled rule: stores the expansion size and origin name
|
|
6
10
|
/// so we never touch Python Rule objects during parsing.
|
|
7
11
|
struct CompiledRule {
|
|
@@ -143,6 +147,7 @@ impl CompiledParseTable {
|
|
|
143
147
|
/// Takes state_stack and value_stack as mutable Python lists,
|
|
144
148
|
/// the token, and is_end flag.
|
|
145
149
|
/// Returns None on shift, or the final value on successful end.
|
|
150
|
+
#[allow(clippy::too_many_arguments)]
|
|
146
151
|
fn feed_token(
|
|
147
152
|
&self,
|
|
148
153
|
state_stack: &Bound<'_, PyList>,
|
|
@@ -152,6 +157,109 @@ impl CompiledParseTable {
|
|
|
152
157
|
is_end: bool,
|
|
153
158
|
parser_state: &Bound<'_, PyAny>,
|
|
154
159
|
py: Python<'_>,
|
|
160
|
+
) -> PyResult<Option<Py<PyAny>>> {
|
|
161
|
+
self.feed_token_inner(
|
|
162
|
+
state_stack,
|
|
163
|
+
value_stack,
|
|
164
|
+
token,
|
|
165
|
+
token_type,
|
|
166
|
+
is_end,
|
|
167
|
+
parser_state,
|
|
168
|
+
py,
|
|
169
|
+
)
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
/// Run the full lex-parse loop in Rust for the BasicLexer path.
|
|
173
|
+
/// Calls next_token and feed_token in a tight Rust loop, only
|
|
174
|
+
/// crossing back to Python for callbacks.
|
|
175
|
+
#[allow(clippy::too_many_arguments)]
|
|
176
|
+
fn parse_loop(
|
|
177
|
+
&self,
|
|
178
|
+
lexer_obj: &Bound<'_, PyAny>,
|
|
179
|
+
lexer_state_obj: &Bound<'_, PyAny>,
|
|
180
|
+
state_stack: &Bound<'_, PyList>,
|
|
181
|
+
value_stack: &Bound<'_, PyList>,
|
|
182
|
+
parser_state: &Bound<'_, PyAny>,
|
|
183
|
+
py: Python<'_>,
|
|
184
|
+
) -> PyResult<Py<PyAny>> {
|
|
185
|
+
let mut last_token: Option<Token> = None;
|
|
186
|
+
|
|
187
|
+
// Tight lex-parse loop: next_token + feed_token without Python overhead
|
|
188
|
+
loop {
|
|
189
|
+
let result = {
|
|
190
|
+
let mut lexer = lexer_obj.downcast::<BasicLexer>()?.borrow_mut();
|
|
191
|
+
let mut lexer_state = lexer_state_obj.downcast::<LexerState>()?.borrow_mut();
|
|
192
|
+
lexer.next_token(&mut lexer_state, parser_state, py)
|
|
193
|
+
};
|
|
194
|
+
match result {
|
|
195
|
+
Ok(token) => {
|
|
196
|
+
let token_type = token.type_.clone();
|
|
197
|
+
let token_py = token.clone().into_pyobject(py)?.into_any();
|
|
198
|
+
last_token = Some(token);
|
|
199
|
+
self.feed_token_inner(
|
|
200
|
+
state_stack,
|
|
201
|
+
value_stack,
|
|
202
|
+
&token_py,
|
|
203
|
+
&token_type,
|
|
204
|
+
false,
|
|
205
|
+
parser_state,
|
|
206
|
+
py,
|
|
207
|
+
)?;
|
|
208
|
+
}
|
|
209
|
+
Err(e) if e.is_instance_of::<pyo3::exceptions::PyEOFError>(py) => {
|
|
210
|
+
break;
|
|
211
|
+
}
|
|
212
|
+
Err(e) => {
|
|
213
|
+
return Err(e);
|
|
214
|
+
}
|
|
215
|
+
}
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
// Feed $END token
|
|
219
|
+
let end_token = if let Some(ref lt) = last_token {
|
|
220
|
+
Token::new(
|
|
221
|
+
"$END".to_string(),
|
|
222
|
+
String::new(),
|
|
223
|
+
lt.start_pos,
|
|
224
|
+
lt.line,
|
|
225
|
+
lt.column,
|
|
226
|
+
lt.end_line,
|
|
227
|
+
lt.end_column,
|
|
228
|
+
lt.end_pos,
|
|
229
|
+
)
|
|
230
|
+
} else {
|
|
231
|
+
Token::new("$END".to_string(), String::new(), 0, 1, 1, None, None, None)
|
|
232
|
+
};
|
|
233
|
+
let end_token_py = end_token.into_pyobject(py)?.into_any();
|
|
234
|
+
|
|
235
|
+
let result = self.feed_token_inner(
|
|
236
|
+
state_stack,
|
|
237
|
+
value_stack,
|
|
238
|
+
&end_token_py,
|
|
239
|
+
"$END",
|
|
240
|
+
true,
|
|
241
|
+
parser_state,
|
|
242
|
+
py,
|
|
243
|
+
)?;
|
|
244
|
+
|
|
245
|
+
result.ok_or_else(|| {
|
|
246
|
+
pyo3::exceptions::PyRuntimeError::new_err("parse_loop: feed_token($END) returned None")
|
|
247
|
+
})
|
|
248
|
+
}
|
|
249
|
+
}
|
|
250
|
+
|
|
251
|
+
impl CompiledParseTable {
|
|
252
|
+
/// Shared feed_token implementation used by both feed_token and parse_loop.
|
|
253
|
+
#[allow(clippy::too_many_arguments)]
|
|
254
|
+
fn feed_token_inner(
|
|
255
|
+
&self,
|
|
256
|
+
state_stack: &Bound<'_, PyList>,
|
|
257
|
+
value_stack: &Bound<'_, PyList>,
|
|
258
|
+
token: &Bound<'_, PyAny>,
|
|
259
|
+
token_type: &str,
|
|
260
|
+
is_end: bool,
|
|
261
|
+
parser_state: &Bound<'_, PyAny>,
|
|
262
|
+
py: Python<'_>,
|
|
155
263
|
) -> PyResult<Option<Py<PyAny>>> {
|
|
156
264
|
loop {
|
|
157
265
|
let stack_len = state_stack.len();
|
|
@@ -209,7 +317,7 @@ impl CompiledParseTable {
|
|
|
209
317
|
value_stack.del_slice(start, vstack_len)?;
|
|
210
318
|
slice
|
|
211
319
|
} else {
|
|
212
|
-
PyList::empty(py)
|
|
320
|
+
PyList::empty(py)
|
|
213
321
|
};
|
|
214
322
|
|
|
215
323
|
let value = rule.callback.call1(py, (&children,))?;
|
|
@@ -6,6 +6,9 @@ pub struct Scanner {
|
|
|
6
6
|
combined_re: Option<Regex>,
|
|
7
7
|
individual_patterns: Vec<(String, Regex)>,
|
|
8
8
|
group_to_terminal: Vec<String>,
|
|
9
|
+
/// Capture-group index of each terminal in `combined_re`. Terminal
|
|
10
|
+
/// patterns may contain their own groups, so indices are not contiguous.
|
|
11
|
+
group_indices: Vec<usize>,
|
|
9
12
|
}
|
|
10
13
|
|
|
11
14
|
impl Scanner {
|
|
@@ -27,16 +30,33 @@ impl Scanner {
|
|
|
27
30
|
if let Ok(re) = Regex::new(pattern) {
|
|
28
31
|
individual_patterns.push((name.clone(), re));
|
|
29
32
|
}
|
|
30
|
-
pattern_parts.push(format!("({})", pattern));
|
|
33
|
+
pattern_parts.push(format!("(?P<{}>{})", name, pattern));
|
|
31
34
|
group_to_terminal.push(name.clone());
|
|
32
35
|
}
|
|
33
36
|
|
|
34
37
|
let combined_re = Regex::new(&pattern_parts.join("|")).ok();
|
|
35
38
|
|
|
39
|
+
let group_indices = match combined_re {
|
|
40
|
+
Some(ref re) => {
|
|
41
|
+
let names: Vec<Option<&str>> = re.capture_names().collect();
|
|
42
|
+
group_to_terminal
|
|
43
|
+
.iter()
|
|
44
|
+
.map(|n| {
|
|
45
|
+
names
|
|
46
|
+
.iter()
|
|
47
|
+
.position(|g| *g == Some(n.as_str()))
|
|
48
|
+
.unwrap_or(0)
|
|
49
|
+
})
|
|
50
|
+
.collect()
|
|
51
|
+
}
|
|
52
|
+
None => Vec::new(),
|
|
53
|
+
};
|
|
54
|
+
|
|
36
55
|
Ok(Scanner {
|
|
37
56
|
combined_re,
|
|
38
57
|
individual_patterns,
|
|
39
58
|
group_to_terminal,
|
|
59
|
+
group_indices,
|
|
40
60
|
})
|
|
41
61
|
}
|
|
42
62
|
|
|
@@ -54,9 +74,9 @@ impl Scanner {
|
|
|
54
74
|
if full.start() == 0 {
|
|
55
75
|
let match_end = full.end();
|
|
56
76
|
let matched = &text[pos..pos + match_end];
|
|
57
|
-
// Find which group matched
|
|
58
|
-
for (
|
|
59
|
-
if caps.get(
|
|
77
|
+
// Find which terminal's group matched
|
|
78
|
+
for (name, &idx) in self.group_to_terminal.iter().zip(&self.group_indices) {
|
|
79
|
+
if caps.get(idx).is_some_and(|m| m.start() == 0) {
|
|
60
80
|
return Some((matched, name.clone()));
|
|
61
81
|
}
|
|
62
82
|
}
|
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
"""Regression tests for https://github.com/robertozimek/lark-rust/issues/1"""
|
|
2
|
+
import pytest
|
|
3
|
+
from lark import Lark
|
|
4
|
+
from lark.exceptions import UnexpectedCharacters, UnexpectedToken
|
|
5
|
+
import lark_rust
|
|
6
|
+
|
|
7
|
+
GRAMMAR = r"""
|
|
8
|
+
INT_NUMBER: /\d+/
|
|
9
|
+
FLOAT_NUMBER: /\d*\.\d+|\d+e(\+|\-)\d+/
|
|
10
|
+
VARIABLE: /[a-zA-Z_]\w*/
|
|
11
|
+
NEWLINE: "\n"
|
|
12
|
+
PLUS: "+"
|
|
13
|
+
EQ: "="
|
|
14
|
+
LPAR: "("
|
|
15
|
+
RPAR: ")"
|
|
16
|
+
LBRACK: "["
|
|
17
|
+
RBRACK: "]"
|
|
18
|
+
COMMA: ","
|
|
19
|
+
TYPE_COMPLEX: "complex"
|
|
20
|
+
|
|
21
|
+
%ignore /[ \t]+/
|
|
22
|
+
|
|
23
|
+
start: (NEWLINE* instruction)+ NEWLINE*
|
|
24
|
+
|
|
25
|
+
instruction: expr
|
|
26
|
+
| VARIABLE EQ expr
|
|
27
|
+
|
|
28
|
+
expr: expr PLUS atom
|
|
29
|
+
| atom
|
|
30
|
+
|
|
31
|
+
atom: INT_NUMBER
|
|
32
|
+
| FLOAT_NUMBER
|
|
33
|
+
| VARIABLE
|
|
34
|
+
| TYPE_COMPLEX LPAR expr COMMA expr RPAR
|
|
35
|
+
| LBRACK expr RBRACK
|
|
36
|
+
"""
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def _parsers():
|
|
40
|
+
kw = dict(parser="lalr", lexer="contextual")
|
|
41
|
+
return Lark(GRAMMAR, **kw), Lark(GRAMMAR, _plugins=lark_rust.plugins, **kw)
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def _tokens(tree):
|
|
45
|
+
return [(t.type, t.value) for t in tree.scan_values(lambda _: True)]
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
@pytest.mark.parametrize("text", [
|
|
49
|
+
"complex_number = complex(2,3)",
|
|
50
|
+
"complexity + complex(1, 2.5)",
|
|
51
|
+
"x = 1e+5 + [y]",
|
|
52
|
+
"a = 1\nb = a + 2\n",
|
|
53
|
+
])
|
|
54
|
+
def test_matches_lark(text):
|
|
55
|
+
ref, rust = _parsers()
|
|
56
|
+
assert _tokens(rust.parse(text)) == _tokens(ref.parse(text))
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def test_keyword_prefix_lexed_as_variable():
|
|
60
|
+
_, rust = _parsers()
|
|
61
|
+
tokens = _tokens(rust.parse("complex_number = complex(2,3)"))
|
|
62
|
+
assert tokens[0] == ("VARIABLE", "complex_number")
|
|
63
|
+
assert tokens[2] == ("TYPE_COMPLEX", "complex")
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def test_terminal_with_inner_group_does_not_shift_types():
|
|
67
|
+
# FLOAT_NUMBER contains a capture group; tokens after it must keep their types.
|
|
68
|
+
_, rust = _parsers()
|
|
69
|
+
tokens = _tokens(rust.parse("x = 1e+5 + [y]"))
|
|
70
|
+
assert ("FLOAT_NUMBER", "1e+5") in tokens
|
|
71
|
+
assert ("LBRACK", "[") in tokens
|
|
72
|
+
assert ("EQ", "=") in tokens
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def test_basic_lexer_exposes_terminals_by_name():
|
|
76
|
+
_, rust = _parsers()
|
|
77
|
+
root = rust.parser.lexer.root_lexer
|
|
78
|
+
assert "VARIABLE" in root.terminals_by_name
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def test_unexpected_token_from_root_lexer():
|
|
82
|
+
# "=" is valid for the root lexer but not after EQ, so the contextual
|
|
83
|
+
# lexer raises UnexpectedToken built from root_lexer.terminals_by_name.
|
|
84
|
+
ref, rust = _parsers()
|
|
85
|
+
with pytest.raises(UnexpectedToken) as ref_exc:
|
|
86
|
+
ref.parse("x = = 1")
|
|
87
|
+
with pytest.raises(UnexpectedToken) as rust_exc:
|
|
88
|
+
rust.parse("x = = 1")
|
|
89
|
+
assert rust_exc.value.token.type == ref_exc.value.token.type == "EQ"
|
|
90
|
+
assert rust_exc.value.expected == ref_exc.value.expected
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def test_unexpected_characters():
|
|
94
|
+
_, rust = _parsers()
|
|
95
|
+
with pytest.raises(UnexpectedCharacters):
|
|
96
|
+
rust.parse("x = 1 $ 2")
|
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
"""Exact reproduction from https://github.com/robertozimek/lark-rust/issues/1"""
|
|
2
|
+
from lark import Lark
|
|
3
|
+
import lark_rust
|
|
4
|
+
|
|
5
|
+
GRAMMAR = r"""// ==================== TOKENS ====================
|
|
6
|
+
INT_NUMBER: /\d+/
|
|
7
|
+
FLOAT_NUMBER: /\d*\.\d+|\d+e(\+|\-)\d+/
|
|
8
|
+
VARIABLE: /[a-zA-Z_]\w*/
|
|
9
|
+
NEWLINE: "\n"
|
|
10
|
+
PLUS: "+"
|
|
11
|
+
MINUS: "-"
|
|
12
|
+
MUL: "*"
|
|
13
|
+
DIV: "/"
|
|
14
|
+
MOD: "%"
|
|
15
|
+
POWER: "**"
|
|
16
|
+
EQ: "="
|
|
17
|
+
LPAR: "("
|
|
18
|
+
RPAR: ")"
|
|
19
|
+
LBRACK: "["
|
|
20
|
+
RBRACK: "]"
|
|
21
|
+
COMMA: ","
|
|
22
|
+
COLON: ":"
|
|
23
|
+
TYPE_COMPLEX: "complex"
|
|
24
|
+
TYPE_FLOAT: "float"
|
|
25
|
+
TYPE_INT: "int"
|
|
26
|
+
TYPE_VECTOR: "vector"
|
|
27
|
+
SUM_KEYWORD: "sum"
|
|
28
|
+
MEAN_KEYWORD: "mean"
|
|
29
|
+
DOT_KEYWORD: "dot"
|
|
30
|
+
PRINT_KEYWORD: "print"
|
|
31
|
+
|
|
32
|
+
%ignore /[ \t]+/
|
|
33
|
+
%ignore /\/\/.*\n/
|
|
34
|
+
|
|
35
|
+
start: vec_lang_program
|
|
36
|
+
|
|
37
|
+
vec_lang_program: vec_lang_instructions_sequence
|
|
38
|
+
|
|
39
|
+
vec_lang_instructions_sequence: (NEWLINE* vec_lang_instruction)+ NEWLINE*
|
|
40
|
+
|
|
41
|
+
vec_lang_instruction: arithmetic_expression_level_1
|
|
42
|
+
| function_decl
|
|
43
|
+
| variable_expression EQ arithmetic_expression_level_1
|
|
44
|
+
| PRINT_KEYWORD LPAR function_args RPAR
|
|
45
|
+
|
|
46
|
+
arithmetic_expression_level_1: arithmetic_expression_level_1 PLUS arithmetic_expression_level_2
|
|
47
|
+
| arithmetic_expression_level_1 MINUS arithmetic_expression_level_2
|
|
48
|
+
| arithmetic_expression_level_2
|
|
49
|
+
|
|
50
|
+
arithmetic_expression_level_2: arithmetic_expression_level_2 MUL arithmetic_expression_level_3
|
|
51
|
+
| arithmetic_expression_level_2 DIV arithmetic_expression_level_3
|
|
52
|
+
| arithmetic_expression_level_2 MOD arithmetic_expression_level_3
|
|
53
|
+
| arithmetic_expression_level_3
|
|
54
|
+
|
|
55
|
+
arithmetic_expression_level_3: arithmetic_expression_level_3 POWER arithmetic_expression_level_4
|
|
56
|
+
| arithmetic_expression_level_4
|
|
57
|
+
|
|
58
|
+
arithmetic_expression_level_4: number_expression
|
|
59
|
+
| variable_expression
|
|
60
|
+
| vector
|
|
61
|
+
| indexing
|
|
62
|
+
| function_call
|
|
63
|
+
| LPAR arithmetic_expression_level_1 RPAR
|
|
64
|
+
|
|
65
|
+
number_expression: number
|
|
66
|
+
| complex_number
|
|
67
|
+
|
|
68
|
+
number: INT_NUMBER
|
|
69
|
+
| FLOAT_NUMBER
|
|
70
|
+
| PLUS INT_NUMBER
|
|
71
|
+
| MINUS INT_NUMBER
|
|
72
|
+
| PLUS FLOAT_NUMBER
|
|
73
|
+
| MINUS FLOAT_NUMBER
|
|
74
|
+
|
|
75
|
+
complex_number: TYPE_COMPLEX LPAR number COMMA number RPAR
|
|
76
|
+
| number VARIABLE
|
|
77
|
+
|
|
78
|
+
variable_expression: VARIABLE
|
|
79
|
+
|
|
80
|
+
vector: LBRACK components RBRACK
|
|
81
|
+
| LBRACK range RBRACK
|
|
82
|
+
| slicing
|
|
83
|
+
|
|
84
|
+
components: arithmetic_expression_level_1
|
|
85
|
+
| components COMMA arithmetic_expression_level_1
|
|
86
|
+
|
|
87
|
+
range: INT_NUMBER COLON INT_NUMBER
|
|
88
|
+
| MINUS INT_NUMBER COLON INT_NUMBER
|
|
89
|
+
| INT_NUMBER COLON MINUS INT_NUMBER
|
|
90
|
+
| MINUS INT_NUMBER COLON MINUS INT_NUMBER
|
|
91
|
+
|
|
92
|
+
indexing: variable_expression LBRACK INT_NUMBER RBRACK
|
|
93
|
+
| vector LBRACK INT_NUMBER RBRACK
|
|
94
|
+
|
|
95
|
+
slicing: variable_expression LBRACK range RBRACK
|
|
96
|
+
| vector LBRACK range RBRACK
|
|
97
|
+
|
|
98
|
+
function_call: variable_expression LPAR function_args RPAR
|
|
99
|
+
| SUM_KEYWORD LPAR function_args RPAR
|
|
100
|
+
| MEAN_KEYWORD LPAR function_args RPAR
|
|
101
|
+
| DOT_KEYWORD LPAR function_args RPAR
|
|
102
|
+
|
|
103
|
+
function_args: arithmetic_expression_level_1
|
|
104
|
+
| function_args COMMA arithmetic_expression_level_1
|
|
105
|
+
|
|
106
|
+
function_decl: variable_expression LPAR function_decl_args RPAR EQ arithmetic_expression_level_1
|
|
107
|
+
|
|
108
|
+
function_decl_args: variable_expression COLON type
|
|
109
|
+
| function_decl_args COMMA variable_expression COLON type
|
|
110
|
+
|
|
111
|
+
type: TYPE_COMPLEX
|
|
112
|
+
| TYPE_FLOAT
|
|
113
|
+
| TYPE_INT
|
|
114
|
+
| TYPE_VECTOR
|
|
115
|
+
"""
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
def test_issue_1_exact_repro():
|
|
119
|
+
text = 'complex_number = complex(2,3)'
|
|
120
|
+
parser = Lark(GRAMMAR, parser='lalr', lexer='contextual', _plugins=lark_rust.plugins)
|
|
121
|
+
tree = parser.parse(text)
|
|
122
|
+
|
|
123
|
+
expected = Lark(GRAMMAR, parser='lalr', lexer='contextual').parse(text)
|
|
124
|
+
assert tree == expected
|
|
@@ -1,76 +0,0 @@
|
|
|
1
|
-
name: Release
|
|
2
|
-
|
|
3
|
-
on:
|
|
4
|
-
push:
|
|
5
|
-
tags:
|
|
6
|
-
- "v*"
|
|
7
|
-
|
|
8
|
-
permissions:
|
|
9
|
-
contents: read
|
|
10
|
-
|
|
11
|
-
jobs:
|
|
12
|
-
build-sdist:
|
|
13
|
-
runs-on: ubuntu-latest
|
|
14
|
-
steps:
|
|
15
|
-
- uses: actions/checkout@v4
|
|
16
|
-
- name: Build sdist
|
|
17
|
-
uses: PyO3/maturin-action@v1
|
|
18
|
-
with:
|
|
19
|
-
command: sdist
|
|
20
|
-
args: --out dist
|
|
21
|
-
- uses: actions/upload-artifact@v4
|
|
22
|
-
with:
|
|
23
|
-
name: sdist
|
|
24
|
-
path: dist/*.tar.gz
|
|
25
|
-
|
|
26
|
-
build-wheels:
|
|
27
|
-
runs-on: ${{ matrix.os }}
|
|
28
|
-
strategy:
|
|
29
|
-
fail-fast: false
|
|
30
|
-
matrix:
|
|
31
|
-
include:
|
|
32
|
-
# Linux x86_64
|
|
33
|
-
- os: ubuntu-latest
|
|
34
|
-
target: x86_64
|
|
35
|
-
# Linux aarch64
|
|
36
|
-
- os: ubuntu-latest
|
|
37
|
-
target: aarch64
|
|
38
|
-
# macOS universal2 (x86_64 + arm64 fat binary)
|
|
39
|
-
- os: macos-latest
|
|
40
|
-
target: universal2-apple-darwin
|
|
41
|
-
# Windows x86_64
|
|
42
|
-
- os: windows-latest
|
|
43
|
-
target: x64
|
|
44
|
-
steps:
|
|
45
|
-
- uses: actions/checkout@v4
|
|
46
|
-
- uses: actions/setup-python@v5
|
|
47
|
-
with:
|
|
48
|
-
python-version: "3.x"
|
|
49
|
-
- name: Build wheels
|
|
50
|
-
uses: PyO3/maturin-action@v1
|
|
51
|
-
with:
|
|
52
|
-
target: ${{ matrix.target }}
|
|
53
|
-
args: --release --out dist --find-interpreter
|
|
54
|
-
sccache: "true"
|
|
55
|
-
manylinux: auto
|
|
56
|
-
- uses: actions/upload-artifact@v4
|
|
57
|
-
with:
|
|
58
|
-
name: wheels-${{ matrix.os }}-${{ matrix.target }}
|
|
59
|
-
path: dist/*.whl
|
|
60
|
-
|
|
61
|
-
publish:
|
|
62
|
-
needs: [build-sdist, build-wheels]
|
|
63
|
-
runs-on: ubuntu-latest
|
|
64
|
-
environment: pypi
|
|
65
|
-
permissions:
|
|
66
|
-
id-token: write
|
|
67
|
-
steps:
|
|
68
|
-
- uses: actions/download-artifact@v4
|
|
69
|
-
with:
|
|
70
|
-
path: dist
|
|
71
|
-
merge-multiple: true
|
|
72
|
-
- name: Publish to PyPI
|
|
73
|
-
uses: PyO3/maturin-action@v1
|
|
74
|
-
with:
|
|
75
|
-
command: upload
|
|
76
|
-
args: --non-interactive --skip-existing dist/*
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|