sqlparse-rs 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,52 @@
1
+ name: CI
2
+
3
+ on:
4
+ push:
5
+ branches: [main]
6
+ pull_request:
7
+
8
+ jobs:
9
+ lint:
10
+ runs-on: ubuntu-latest
11
+ steps:
12
+ - uses: actions/checkout@v4
13
+ - uses: dtolnay/rust-toolchain@stable
14
+ with:
15
+ components: rustfmt, clippy
16
+ - run: cargo fmt --check
17
+ - run: cargo clippy --release -- -D warnings
18
+
19
+ test:
20
+ runs-on: ${{ matrix.os }}
21
+ strategy:
22
+ fail-fast: false
23
+ matrix:
24
+ os: [ubuntu-24.04, macos-latest, windows-latest]
25
+ python: ["3.10", "3.13"]
26
+ env:
27
+ PYTHONUTF8: "1"
28
+ steps:
29
+ - uses: actions/checkout@v4
30
+ - uses: actions/setup-python@v5
31
+ with:
32
+ python-version: ${{ matrix.python }}
33
+ - uses: dtolnay/rust-toolchain@stable
34
+ - name: Build wheel and install
35
+ shell: bash
36
+ run: |
37
+ python -m pip install maturin pytest "sqlparse==0.6.1"
38
+ maturin build --release -o dist
39
+ python -m pip install --no-index --find-links dist sqlparse-rs
40
+ - name: Unit tests
41
+ run: python -m pytest tests/test_basic.py -q
42
+ - name: Differential test against sqlparse
43
+ shell: bash
44
+ run: |
45
+ python -m pip download "sqlparse==0.6.1" --no-deps --no-binary :all: -d upstream
46
+ mkdir upstream/src && tar xzf upstream/sqlparse-0.6.1.tar.gz -C upstream/src
47
+ python -I tests/diff_test.py upstream/src/sqlparse-0.6.1/tests/files --n-fuzz 20000
48
+ - name: sqlparse's own test suite with the Rust lexer enabled
49
+ shell: bash
50
+ run: |
51
+ cd upstream/src/sqlparse-0.6.1
52
+ python -c "import sqlparse_rs, sys, pytest; sqlparse_rs.patch(); sys.exit(pytest.main(['-q', 'tests']))"
@@ -0,0 +1,61 @@
1
+ # Builds wheels + sdist and publishes to PyPI when a tag like v0.1.0 is pushed.
2
+ # Nothing here runs until you push such a tag. One-time setup before the first
3
+ # release: on PyPI, add a "trusted publisher" for this repository (workflow:
4
+ # release.yml, environment: pypi). No API token needs to be stored.
5
+ name: Release
6
+
7
+ on:
8
+ push:
9
+ tags: ["v*"]
10
+
11
+ permissions:
12
+ contents: read
13
+
14
+ jobs:
15
+ wheels:
16
+ runs-on: ${{ matrix.os }}
17
+ strategy:
18
+ matrix:
19
+ include:
20
+ - { os: ubuntu-latest, target: x86_64 }
21
+ - { os: ubuntu-latest, target: aarch64 }
22
+ - { os: macos-latest, target: x86_64 }
23
+ - { os: macos-latest, target: aarch64 }
24
+ - { os: windows-latest, target: x64 }
25
+ steps:
26
+ - uses: actions/checkout@v4
27
+ - uses: PyO3/maturin-action@v1
28
+ with:
29
+ target: ${{ matrix.target }}
30
+ args: --release --out dist
31
+ manylinux: auto
32
+ - uses: actions/upload-artifact@v4
33
+ with:
34
+ name: wheels-${{ matrix.os }}-${{ matrix.target }}
35
+ path: dist
36
+
37
+ sdist:
38
+ runs-on: ubuntu-latest
39
+ steps:
40
+ - uses: actions/checkout@v4
41
+ - uses: PyO3/maturin-action@v1
42
+ with:
43
+ command: sdist
44
+ args: --out dist
45
+ - uses: actions/upload-artifact@v4
46
+ with:
47
+ name: sdist
48
+ path: dist
49
+
50
+ publish:
51
+ needs: [wheels, sdist]
52
+ runs-on: ubuntu-latest
53
+ environment: pypi
54
+ permissions:
55
+ id-token: write
56
+ steps:
57
+ - uses: actions/download-artifact@v4
58
+ with:
59
+ path: dist
60
+ merge-multiple: true
61
+ - uses: pypa/gh-action-pypi-publish@release/v1
@@ -0,0 +1,4 @@
1
+ target/
2
+ dist/
3
+ __pycache__/
4
+ *.egg-info/
@@ -0,0 +1,191 @@
1
+ # This file is automatically @generated by Cargo.
2
+ # It is not intended for manual editing.
3
+ version = 4
4
+
5
+ [[package]]
6
+ name = "crossbeam-deque"
7
+ version = "0.8.8"
8
+ source = "registry+https://github.com/rust-lang/crates.io-index"
9
+ checksum = "622f3fc73690be383c7214310406f28a90e6edeadc3cea882f9d71e495b9711a"
10
+ dependencies = [
11
+ "crossbeam-epoch",
12
+ "crossbeam-utils",
13
+ ]
14
+
15
+ [[package]]
16
+ name = "crossbeam-epoch"
17
+ version = "0.9.21"
18
+ source = "registry+https://github.com/rust-lang/crates.io-index"
19
+ checksum = "dc74980687109a3b14c72fd458107bf0baa1da1a1a805e178d15501ba9b86d9d"
20
+ dependencies = [
21
+ "crossbeam-utils",
22
+ ]
23
+
24
+ [[package]]
25
+ name = "crossbeam-utils"
26
+ version = "0.8.23"
27
+ source = "registry+https://github.com/rust-lang/crates.io-index"
28
+ checksum = "a31eee39dddec8330830986fcd7625edb5a24ec90ea038215273bbc3adb08ac6"
29
+
30
+ [[package]]
31
+ name = "either"
32
+ version = "1.19.0"
33
+ source = "registry+https://github.com/rust-lang/crates.io-index"
34
+ checksum = "0e9c71c2167ca323c882b99918929403426e2373ea17242ff5653e0d5e1058be"
35
+
36
+ [[package]]
37
+ name = "heck"
38
+ version = "0.5.0"
39
+ source = "registry+https://github.com/rust-lang/crates.io-index"
40
+ checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"
41
+
42
+ [[package]]
43
+ name = "libc"
44
+ version = "0.2.191"
45
+ source = "registry+https://github.com/rust-lang/crates.io-index"
46
+ checksum = "65cd87c9a6d60536580b316ad985726104730a5151c99845f2490428ef91240a"
47
+
48
+ [[package]]
49
+ name = "once_cell"
50
+ version = "1.21.4"
51
+ source = "registry+https://github.com/rust-lang/crates.io-index"
52
+ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50"
53
+
54
+ [[package]]
55
+ name = "portable-atomic"
56
+ version = "1.15.0"
57
+ source = "registry+https://github.com/rust-lang/crates.io-index"
58
+ checksum = "05c8b63e8d9609db387f0324918f81d68fe27748f084ef092fb35954d0539a85"
59
+
60
+ [[package]]
61
+ name = "proc-macro2"
62
+ version = "1.0.107"
63
+ source = "registry+https://github.com/rust-lang/crates.io-index"
64
+ checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9"
65
+ dependencies = [
66
+ "unicode-ident",
67
+ ]
68
+
69
+ [[package]]
70
+ name = "pyo3"
71
+ version = "0.30.1"
72
+ source = "registry+https://github.com/rust-lang/crates.io-index"
73
+ checksum = "229ca13b8a15bb0d1487b81ab975bfcd97513416499a8312011b6036e42e0de4"
74
+ dependencies = [
75
+ "libc",
76
+ "once_cell",
77
+ "portable-atomic",
78
+ "pyo3-build-config",
79
+ "pyo3-ffi",
80
+ "pyo3-macros",
81
+ ]
82
+
83
+ [[package]]
84
+ name = "pyo3-build-config"
85
+ version = "0.30.1"
86
+ source = "registry+https://github.com/rust-lang/crates.io-index"
87
+ checksum = "70130e2b6dd1ee589a8788026a429561ca5c40fa7a2fa7914c2282eddae24700"
88
+ dependencies = [
89
+ "target-lexicon",
90
+ ]
91
+
92
+ [[package]]
93
+ name = "pyo3-ffi"
94
+ version = "0.30.1"
95
+ source = "registry+https://github.com/rust-lang/crates.io-index"
96
+ checksum = "5e7ff73c555c286d212d4ce7e5fbda845a973c4f61bd5f160f5f2c3c2a04d1c4"
97
+ dependencies = [
98
+ "libc",
99
+ "pyo3-build-config",
100
+ ]
101
+
102
+ [[package]]
103
+ name = "pyo3-macros"
104
+ version = "0.30.1"
105
+ source = "registry+https://github.com/rust-lang/crates.io-index"
106
+ checksum = "c46d2e5862c7bb8207e3b639f713dda3c499933444784154c11d013c5a691b60"
107
+ dependencies = [
108
+ "proc-macro2",
109
+ "pyo3-macros-backend",
110
+ "quote",
111
+ "syn",
112
+ ]
113
+
114
+ [[package]]
115
+ name = "pyo3-macros-backend"
116
+ version = "0.30.1"
117
+ source = "registry+https://github.com/rust-lang/crates.io-index"
118
+ checksum = "38b89f066f2ae72a2aef52d1537ca2fcd104440ebfadee06e97aa90738a25b9b"
119
+ dependencies = [
120
+ "heck",
121
+ "proc-macro2",
122
+ "quote",
123
+ "syn",
124
+ ]
125
+
126
+ [[package]]
127
+ name = "quote"
128
+ version = "1.0.47"
129
+ source = "registry+https://github.com/rust-lang/crates.io-index"
130
+ checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001"
131
+ dependencies = [
132
+ "proc-macro2",
133
+ ]
134
+
135
+ [[package]]
136
+ name = "rayon"
137
+ version = "1.12.0"
138
+ source = "registry+https://github.com/rust-lang/crates.io-index"
139
+ checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d"
140
+ dependencies = [
141
+ "either",
142
+ "rayon-core",
143
+ ]
144
+
145
+ [[package]]
146
+ name = "rayon-core"
147
+ version = "1.13.0"
148
+ source = "registry+https://github.com/rust-lang/crates.io-index"
149
+ checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91"
150
+ dependencies = [
151
+ "crossbeam-deque",
152
+ "crossbeam-utils",
153
+ ]
154
+
155
+ [[package]]
156
+ name = "rustc-hash"
157
+ version = "2.1.3"
158
+ source = "registry+https://github.com/rust-lang/crates.io-index"
159
+ checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d"
160
+
161
+ [[package]]
162
+ name = "sqlparse-rs"
163
+ version = "0.1.0"
164
+ dependencies = [
165
+ "pyo3",
166
+ "rayon",
167
+ "rustc-hash",
168
+ ]
169
+
170
+ [[package]]
171
+ name = "syn"
172
+ version = "3.0.10"
173
+ source = "registry+https://github.com/rust-lang/crates.io-index"
174
+ checksum = "4ae9ccbea3a7b0b09443036490c8f34a8b1b760fd345e5c0dae0aac718faea79"
175
+ dependencies = [
176
+ "proc-macro2",
177
+ "quote",
178
+ "unicode-ident",
179
+ ]
180
+
181
+ [[package]]
182
+ name = "target-lexicon"
183
+ version = "0.13.6"
184
+ source = "registry+https://github.com/rust-lang/crates.io-index"
185
+ checksum = "8c447be3a2abfa84b2d91680c56600bd6b2873866de27b36ec911be705e291da"
186
+
187
+ [[package]]
188
+ name = "unicode-ident"
189
+ version = "1.0.27"
190
+ source = "registry+https://github.com/rust-lang/crates.io-index"
191
+ checksum = "a2c754d6c33795a1c324727428e5a7dedb5b06195f9890bdbcba760d3e246563"
@@ -0,0 +1,21 @@
1
+ [package]
2
+ name = "sqlparse-rs"
3
+ version = "0.1.0"
4
+ edition = "2021"
5
+ description = "Rust-accelerated lexer and statement splitter for sqlparse"
6
+ license = "BSD-3-Clause"
7
+ readme = "README.md"
8
+
9
+ [lib]
10
+ name = "_core"
11
+ crate-type = ["cdylib"]
12
+
13
+ [dependencies]
14
+ pyo3 = { version = "0.30.1", features = ["extension-module", "abi3-py39"] }
15
+ rayon = "1.12.0"
16
+ rustc-hash = "2"
17
+
18
+ [profile.release]
19
+ opt-level = 3
20
+ lto = "fat"
21
+ codegen-units = 1
@@ -0,0 +1,32 @@
1
+ BSD 3-Clause License
2
+
3
+ Copyright (c) 2026, markocerkez
4
+ All rights reserved.
5
+
6
+ This project is a port of sqlparse (https://github.com/andialbrecht/sqlparse):
7
+
8
+ Copyright (c) 2016, Andi Albrecht <albrecht.andi@gmail.com>
9
+ All rights reserved.
10
+
11
+ Redistribution and use in source and binary forms, with or without modification,
12
+ are permitted provided that the following conditions are met:
13
+
14
+ * Redistributions of source code must retain the above copyright notice,
15
+ this list of conditions and the following disclaimer.
16
+ * Redistributions in binary form must reproduce the above copyright notice,
17
+ this list of conditions and the following disclaimer in the documentation
18
+ and/or other materials provided with the distribution.
19
+ * Neither the name of the authors nor the names of its contributors may be
20
+ used to endorse or promote products derived from this software without
21
+ specific prior written permission.
22
+
23
+ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
24
+ ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
25
+ WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
26
+ DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
27
+ ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
28
+ DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
29
+ SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
30
+ CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
31
+ OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
32
+ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
@@ -0,0 +1,76 @@
1
+ Metadata-Version: 2.4
2
+ Name: sqlparse-rs
3
+ Version: 0.1.0
4
+ Classifier: License :: OSI Approved :: BSD License
5
+ Classifier: Programming Language :: Rust
6
+ Classifier: Programming Language :: Python :: 3
7
+ Requires-Dist: sqlparse>=0.6.1,<0.7
8
+ Requires-Dist: pytest ; extra == 'test'
9
+ Provides-Extra: test
10
+ License-File: LICENSE
11
+ Summary: Rust-accelerated lexer and statement splitter for sqlparse
12
+ Keywords: sql,parser,sqlparse,rust
13
+ License: BSD-3-Clause
14
+ Requires-Python: >=3.10
15
+ Description-Content-Type: text/markdown; charset=UTF-8; variant=GFM
16
+ Project-URL: Repository, https://github.com/markocerkez/sqlparse-rs
17
+ Project-URL: Upstream, https://github.com/andialbrecht/sqlparse
18
+
19
+ # sqlparse-rs
20
+
21
+ Rust-accelerated lexer and statement splitter for [sqlparse](https://github.com/andialbrecht/sqlparse).
22
+ Output is identical to the original; anything the Rust core does not handle is passed to the
23
+ original implementation.
24
+
25
+ ```python
26
+ import sqlparse, sqlparse_rs
27
+ sqlparse_rs.patch() # sqlparse.split / parse / format now use the Rust lexer
28
+ sqlparse.split(script) # ~50-70x faster
29
+ sqlparse_rs.split_many(scripts) # parallel, GIL released
30
+ sqlparse_rs.unpatch()
31
+ ```
32
+
33
+ ## Speed
34
+
35
+ sqlparse 0.6.1, Python 3.13, 2 cores, 467 KB of generated queries plus sqlparse's own sample files
36
+ (`bench/bench.py`):
37
+
38
+ | operation | original | Rust | speedup |
39
+ |---|---:|---:|---:|
40
+ | `lexer.tokenize` (467 KB script) | 702 ms | 39 ms | 18x |
41
+ | `split` (467 KB script) | 1038 ms | 15 ms | 69x |
42
+ | `split` each of 1500 queries | 1118 ms | 22 ms | 50x |
43
+ | `split_many` (6000 queries, parallel) | 4240 ms | 50 ms | 86x |
44
+ | `parse` x500 | 1122 ms | 848 ms | 1.3x |
45
+ | `format(reindent)` x500 | 1557 ms | 1144 ms | 1.4x |
46
+
47
+ `parse` and `format` gain less because most of their time is spent in sqlparse's grouping pass
48
+ and token-object construction, which stay in Python. Porting grouping is the obvious next step.
49
+
50
+ ## What is ported
51
+
52
+ * The whole `SQL_REGEX` rule list, as hand-written matchers that reproduce the backtracking order
53
+ of the original patterns (including quirks such as `12abc` lexing as `1` + `2abc`).
54
+ * Dollar-quoted literals and multiline comments (`find_delimited_spans`).
55
+ * Keyword lookup, built from the live `Lexer` dictionaries.
56
+ * `StatementSplitter` (BEGIN/END, CASE, IF, LOOP, DECLARE blocks, GO, ...) for `split()`.
57
+
58
+ ## Fallbacks (results stay identical)
59
+
60
+ Bytes input, lone surrogates, customised lexers (`set_SQL_REGEX`), non-ASCII words that need
61
+ Python's `str.upper()`, and non-ASCII text when Python's Unicode database differs from the one the
62
+ character tables were generated from (`tools/gen_tables.py`). Keywords added via `add_keywords` are
63
+ picked up automatically.
64
+
65
+ ## Testing
66
+
67
+ * `tests/diff_test.py`: compares token streams (same token-type objects, same values) and
68
+ `split()` with and without `strip_semicolon` against the original on sample files and fuzzed
69
+ input; about 2M cases passed against sqlparse 0.6.1.
70
+ * sqlparse's own test suite passes unchanged with `sqlparse_rs.patch()` applied.
71
+ * `tests/test_basic.py`: unit tests.
72
+
73
+ ## License
74
+
75
+ BSD 3-Clause, same as sqlparse; see `LICENSE`.
76
+
@@ -0,0 +1,57 @@
1
+ # sqlparse-rs
2
+
3
+ Rust-accelerated lexer and statement splitter for [sqlparse](https://github.com/andialbrecht/sqlparse).
4
+ Output is identical to the original; anything the Rust core does not handle is passed to the
5
+ original implementation.
6
+
7
+ ```python
8
+ import sqlparse, sqlparse_rs
9
+ sqlparse_rs.patch() # sqlparse.split / parse / format now use the Rust lexer
10
+ sqlparse.split(script) # ~50-70x faster
11
+ sqlparse_rs.split_many(scripts) # parallel, GIL released
12
+ sqlparse_rs.unpatch()
13
+ ```
14
+
15
+ ## Speed
16
+
17
+ sqlparse 0.6.1, Python 3.13, 2 cores, 467 KB of generated queries plus sqlparse's own sample files
18
+ (`bench/bench.py`):
19
+
20
+ | operation | original | Rust | speedup |
21
+ |---|---:|---:|---:|
22
+ | `lexer.tokenize` (467 KB script) | 702 ms | 39 ms | 18x |
23
+ | `split` (467 KB script) | 1038 ms | 15 ms | 69x |
24
+ | `split` each of 1500 queries | 1118 ms | 22 ms | 50x |
25
+ | `split_many` (6000 queries, parallel) | 4240 ms | 50 ms | 86x |
26
+ | `parse` x500 | 1122 ms | 848 ms | 1.3x |
27
+ | `format(reindent)` x500 | 1557 ms | 1144 ms | 1.4x |
28
+
29
+ `parse` and `format` gain less because most of their time is spent in sqlparse's grouping pass
30
+ and token-object construction, which stay in Python. Porting grouping is the obvious next step.
31
+
32
+ ## What is ported
33
+
34
+ * The whole `SQL_REGEX` rule list, as hand-written matchers that reproduce the backtracking order
35
+ of the original patterns (including quirks such as `12abc` lexing as `1` + `2abc`).
36
+ * Dollar-quoted literals and multiline comments (`find_delimited_spans`).
37
+ * Keyword lookup, built from the live `Lexer` dictionaries.
38
+ * `StatementSplitter` (BEGIN/END, CASE, IF, LOOP, DECLARE blocks, GO, ...) for `split()`.
39
+
40
+ ## Fallbacks (results stay identical)
41
+
42
+ Bytes input, lone surrogates, customised lexers (`set_SQL_REGEX`), non-ASCII words that need
43
+ Python's `str.upper()`, and non-ASCII text when Python's Unicode database differs from the one the
44
+ character tables were generated from (`tools/gen_tables.py`). Keywords added via `add_keywords` are
45
+ picked up automatically.
46
+
47
+ ## Testing
48
+
49
+ * `tests/diff_test.py`: compares token streams (same token-type objects, same values) and
50
+ `split()` with and without `strip_semicolon` against the original on sample files and fuzzed
51
+ input; about 2M cases passed against sqlparse 0.6.1.
52
+ * sqlparse's own test suite passes unchanged with `sqlparse_rs.patch()` applied.
53
+ * `tests/test_basic.py`: unit tests.
54
+
55
+ ## License
56
+
57
+ BSD 3-Clause, same as sqlparse; see `LICENSE`.
@@ -0,0 +1,77 @@
1
+ """Benchmark original sqlparse vs the Rust-backed one.
2
+
3
+ usage: python3 -I bench.py [sql_dir] (sql_dir: directory with *.sql sample files, e.g. sqlparse's tests/files)
4
+ """
5
+ import glob
6
+ import random
7
+ import sys
8
+ import time
9
+
10
+ import sqlparse
11
+ from sqlparse import lexer
12
+
13
+ import sqlparse_rs
14
+
15
+ files = []
16
+ for d in sys.argv[1:]:
17
+ for f in sorted(glob.glob(d + "/*.sql")):
18
+ try:
19
+ files.append(open(f, encoding="utf-8").read())
20
+ except Exception:
21
+ pass
22
+
23
+ rnd = random.Random(7)
24
+ tables = ["orders o", "customers c", "public.invoices i", '"Order Items" oi',
25
+ "(SELECT id, SUM(total) AS t FROM payments GROUP BY id) p"]
26
+ cols = ["o.id", "c.name", "SUM(oi.qty * oi.price) AS revenue", "COALESCE(c.email, 'n/a')",
27
+ "CASE WHEN o.status = 'x' THEN 1 ELSE 0 END AS flag", "i.total::numeric(10,2)"]
28
+ conds = ["o.created_at >= '2024-01-01'", "c.id IN (1,2,3,4)", "o.total BETWEEN 10 AND 100",
29
+ "c.name LIKE 'A%' -- comment", "NOT EXISTS (SELECT 1 FROM refunds r WHERE r.order_id = o.id)"]
30
+
31
+
32
+ def q():
33
+ return ("SELECT " + ", ".join(rnd.sample(cols, rnd.randint(2, 5))) + " FROM " + tables[0]
34
+ + " JOIN " + tables[1] + " ON c.id = o.customer_id "
35
+ + (" LEFT JOIN " + rnd.choice(tables[2:]) + " ON 1=1 " if rnd.random() < .6 else "")
36
+ + " WHERE " + " AND ".join(rnd.sample(conds, rnd.randint(1, 4)))
37
+ + " GROUP BY 1, 2 ORDER BY 1 DESC LIMIT 100;")
38
+
39
+
40
+ gen = [q() for _ in range(1500)]
41
+ script = "\n".join(gen)
42
+ big = "\n".join(files) or script
43
+ print("corpus: %d generated queries (%d KB), sample files %d KB\n" % (len(gen), len(script) // 1000, len(big) // 1000))
44
+
45
+
46
+ def best(fn, reps=5):
47
+ b = 9e9
48
+ for _ in range(reps):
49
+ s = time.perf_counter()
50
+ fn()
51
+ b = min(b, time.perf_counter() - s)
52
+ return b
53
+
54
+
55
+ ops = [
56
+ ("tokenize (script)", lambda: sum(1 for _ in lexer.tokenize(script))),
57
+ ("tokenize (sample files)", lambda: sum(1 for _ in lexer.tokenize(big))),
58
+ ("split (script)", lambda: sqlparse.split(script)),
59
+ ("split (sample files)", lambda: sqlparse.split(big)),
60
+ ("split each query x1500", lambda: [sqlparse.split(x) for x in gen]),
61
+ ("parse x500", lambda: [sqlparse.parse(x) for x in gen[:500]]),
62
+ ("format(reindent) x500", lambda: [sqlparse.format(x, reindent=True, keyword_case="upper") for x in gen[:500]]),
63
+ ]
64
+ res = {}
65
+ for mode in ("orig", "rust"):
66
+ sqlparse_rs.unpatch() if mode == "orig" else sqlparse_rs.patch()
67
+ res[mode] = [best(f) for _, f in ops]
68
+ print("%-26s %9s %9s %8s" % ("operation", "original", "rust", "speedup"))
69
+ for (name, _), a, b in zip(ops, res["orig"], res["rust"]):
70
+ print("%-26s %8.1fms %8.1fms %7.1fx" % (name, a * 1e3, b * 1e3, a / b))
71
+
72
+ # batch API
73
+ sqlparse_rs.unpatch()
74
+ batch = gen * 4
75
+ a = best(lambda: [sqlparse.split(x) for x in batch], 3)
76
+ b = best(lambda: sqlparse_rs.split_many(batch), 3)
77
+ print("%-26s %8.1fms %8.1fms %7.1fx" % ("split_many x%d (parallel)" % len(batch), a * 1e3, b * 1e3, a / b))
@@ -0,0 +1,30 @@
1
+ [build-system]
2
+ requires = ["maturin>=1.5,<2"]
3
+ build-backend = "maturin"
4
+
5
+ [project]
6
+ name = "sqlparse-rs"
7
+ version = "0.1.0"
8
+ description = "Rust-accelerated lexer and statement splitter for sqlparse"
9
+ readme = "README.md"
10
+ license = { text = "BSD-3-Clause" }
11
+ requires-python = ">=3.10"
12
+ dependencies = ["sqlparse>=0.6.1,<0.7"]
13
+ keywords = ["sql", "parser", "sqlparse", "rust"]
14
+ classifiers = [
15
+ "License :: OSI Approved :: BSD License",
16
+ "Programming Language :: Rust",
17
+ "Programming Language :: Python :: 3",
18
+ ]
19
+
20
+ [project.optional-dependencies]
21
+ test = ["pytest"]
22
+
23
+ [project.urls]
24
+ Repository = "https://github.com/markocerkez/sqlparse-rs"
25
+ Upstream = "https://github.com/andialbrecht/sqlparse"
26
+
27
+ [tool.maturin]
28
+ python-source = "python"
29
+ module-name = "sqlparse_rs._core"
30
+ features = ["pyo3/extension-module"]