turbotext 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- turbotext-0.1.0/LICENSE +21 -0
- turbotext-0.1.0/PKG-INFO +666 -0
- turbotext-0.1.0/README.md +619 -0
- turbotext-0.1.0/pyproject.toml +96 -0
- turbotext-0.1.0/setup.cfg +4 -0
- turbotext-0.1.0/setup.py +18 -0
- turbotext-0.1.0/src/turbotext/__init__.py +4 -0
- turbotext-0.1.0/src/turbotext/_fast.c +11877 -0
- turbotext-0.1.0/src/turbotext/_fast.pyi +6 -0
- turbotext-0.1.0/src/turbotext/frontier.py +142 -0
- turbotext-0.1.0/src/turbotext/resolve.py +131 -0
- turbotext-0.1.0/src/turbotext/store.py +184 -0
- turbotext-0.1.0/src/turbotext/trie.py +31 -0
- turbotext-0.1.0/src/turbotext.egg-info/PKG-INFO +666 -0
- turbotext-0.1.0/src/turbotext.egg-info/SOURCES.txt +19 -0
- turbotext-0.1.0/src/turbotext.egg-info/dependency_links.txt +1 -0
- turbotext-0.1.0/src/turbotext.egg-info/top_level.txt +1 -0
- turbotext-0.1.0/tests/test_m0_smoke.py +131 -0
- turbotext-0.1.0/tests/test_m1_exact.py +393 -0
- turbotext-0.1.0/tests/test_m2_metadata_priorities.py +319 -0
- turbotext-0.1.0/tests/test_m3_fuzzy.py +286 -0
turbotext-0.1.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Nishank Mahore
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
turbotext-0.1.0/PKG-INFO
ADDED
|
@@ -0,0 +1,666 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: turbotext
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Boundary-aware fuzzy keyword matching with pluggable conflict resolution
|
|
5
|
+
Author-email: Nishank Mahore <nishankmahore@gmail.com>
|
|
6
|
+
Maintainer-email: Nishank Mahore <nishankmahore@gmail.com>
|
|
7
|
+
License: MIT License
|
|
8
|
+
|
|
9
|
+
Copyright (c) 2026 Nishank Mahore
|
|
10
|
+
|
|
11
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
12
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
13
|
+
in the Software without restriction, including without limitation the rights
|
|
14
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
15
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
16
|
+
furnished to do so, subject to the following conditions:
|
|
17
|
+
|
|
18
|
+
The above copyright notice and this permission notice shall be included in all
|
|
19
|
+
copies or substantial portions of the Software.
|
|
20
|
+
|
|
21
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
22
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
23
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
24
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
25
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
26
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
27
|
+
SOFTWARE.
|
|
28
|
+
|
|
29
|
+
Keywords: keyword extraction,fuzzy matching,nlp,text processing,trie,levenshtein,named entity
|
|
30
|
+
Classifier: Development Status :: 3 - Alpha
|
|
31
|
+
Classifier: Intended Audience :: Developers
|
|
32
|
+
Classifier: Intended Audience :: Science/Research
|
|
33
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
34
|
+
Classifier: Programming Language :: Python :: 3
|
|
35
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
36
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
37
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
38
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
39
|
+
Classifier: Programming Language :: Cython
|
|
40
|
+
Classifier: Topic :: Text Processing :: Linguistic
|
|
41
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
42
|
+
Classifier: Typing :: Typed
|
|
43
|
+
Requires-Python: >=3.10
|
|
44
|
+
Description-Content-Type: text/markdown
|
|
45
|
+
License-File: LICENSE
|
|
46
|
+
Dynamic: license-file
|
|
47
|
+
|
|
48
|
+
<p align="center">
|
|
49
|
+
<img src="assets/logo.png" alt="TurboText" width="360"/>
|
|
50
|
+
</p>
|
|
51
|
+
|
|
52
|
+
<h1 align="center">TurboText</h1>
|
|
53
|
+
|
|
54
|
+
<p align="center">
|
|
55
|
+
<b>Lightning-fast, boundary-aware keyword matching for Python</b><br/>
|
|
56
|
+
Exact · Fuzzy · Multi-word · Unicode · Pluggable conflict resolution
|
|
57
|
+
</p>
|
|
58
|
+
|
|
59
|
+
<p align="center">
|
|
60
|
+
<a href="https://pypi.org/project/turbotext/"><img src="https://img.shields.io/badge/PyPI-0.1.0-blue?logo=pypi&logoColor=white" alt="PyPI"/></a>
|
|
61
|
+
<img src="https://img.shields.io/badge/python-3.10%20|%203.11%20|%203.12%20|%203.13-blue?logo=python&logoColor=white" alt="Python"/>
|
|
62
|
+
<a href="LICENSE"><img src="https://img.shields.io/badge/license-MIT-green" alt="License"/></a>
|
|
63
|
+
<a href="https://github.com/nishankmahore/TurboText/actions/workflows/ci.yml"><img src="https://img.shields.io/badge/CI-passing-brightgreen?logo=github" alt="CI"/></a>
|
|
64
|
+
<img src="https://img.shields.io/badge/typed-yes-informational" alt="Typed"/>
|
|
65
|
+
<img src="https://img.shields.io/badge/Cython-optional_fast_path-orange" alt="Cython"/>
|
|
66
|
+
</p>
|
|
67
|
+
|
|
68
|
+
---
|
|
69
|
+
|
|
70
|
+
> TurboText finds keywords in text the way a human editor would — enforcing real word boundaries, tolerating typos, handling multi-word phrases, and letting you declare which match wins when keywords overlap, including an **optimal** (not just greedy) weighted resolver that no other keyword library provides.
|
|
71
|
+
|
|
72
|
+
---
|
|
73
|
+
|
|
74
|
+
## Table of Contents
|
|
75
|
+
|
|
76
|
+
- [Why TurboText?](#why-turbotext)
|
|
77
|
+
- [Features](#features)
|
|
78
|
+
- [Installation](#installation)
|
|
79
|
+
- [Quick Start](#quick-start)
|
|
80
|
+
- [Adding Keywords](#adding-keywords)
|
|
81
|
+
- [Extracting Matches](#extracting-matches)
|
|
82
|
+
- [Replacing Keywords](#replacing-keywords)
|
|
83
|
+
- [Fuzzy Matching](#fuzzy-matching)
|
|
84
|
+
- [Conflict Resolution Policies](#conflict-resolution-policies)
|
|
85
|
+
- [Per-Keyword Metadata](#per-keyword-metadata)
|
|
86
|
+
- [Word Boundary Rules](#word-boundary-rules)
|
|
87
|
+
- [Performance](#performance)
|
|
88
|
+
- [API Reference](#api-reference)
|
|
89
|
+
- [Development](#development)
|
|
90
|
+
- [Author](#author)
|
|
91
|
+
- [License](#license)
|
|
92
|
+
|
|
93
|
+
---
|
|
94
|
+
|
|
95
|
+
## Why TurboText?
|
|
96
|
+
|
|
97
|
+
| | `re` | FlashText | RapidFuzz | **TurboText** |
|
|
98
|
+
|---|:---:|:---:|:---:|:---:|
|
|
99
|
+
| Exact keyword extraction | ✅ | ✅ | ❌ | ✅ |
|
|
100
|
+
| Fuzzy / typo-tolerant | ❌ | ❌ | ✅ | ✅ |
|
|
101
|
+
| Word boundary enforcement | ✅ | ✅ | ❌ | ✅ |
|
|
102
|
+
| Extracts spans from running text | ✅ | ✅ | ❌ | ✅ |
|
|
103
|
+
| Multi-word phrases (native) | ✅ | ✅ | ⚠️ | ✅ |
|
|
104
|
+
| O(text) scaling with vocab | ❌ | ✅ | ❌ | ✅ |
|
|
105
|
+
| Per-keyword metadata | ❌ | ❌ | ❌ | ✅ |
|
|
106
|
+
| Conflict resolution policies | ❌ | ❌ | ❌ | ✅ |
|
|
107
|
+
| Optimal weighted resolution | ❌ | ❌ | ❌ | ✅ |
|
|
108
|
+
|
|
109
|
+
---
|
|
110
|
+
|
|
111
|
+
## Features
|
|
112
|
+
|
|
113
|
+
- **Exact matching** — trie-based O(n) scan, constant in vocabulary size
|
|
114
|
+
- **Fuzzy matching** — Levenshtein distance up to *k* via bounded-edit frontier search
|
|
115
|
+
- **Word boundary enforcement** — Unicode-correct; matches `cat` but not `cats`, `scat`, or `concatenate`
|
|
116
|
+
- **Multi-word keywords** — `"new york"`, `"product manager"` work out of the box
|
|
117
|
+
- **Bulk loading** — accepts `list`, `dict[str, str]`, or `dict[str, list[str]]`
|
|
118
|
+
- **Five conflict policies** — `ALL_OVERLAPS`, `LEFTMOST_LONGEST`, `LEFTMOST_FIRST`, `HIGHEST_PRIORITY`, `OPTIMAL_WEIGHTED`
|
|
119
|
+
- **Optimal resolution** — O(n log n) weighted interval scheduling DP — picks the globally best non-overlapping set
|
|
120
|
+
- **Rich match objects** — canonical form, offsets, edit distance, category, priority, custom metadata
|
|
121
|
+
- **Cython fast-path** — optional compiled extension brings exact-match throughput to FlashText parity
|
|
122
|
+
|
|
123
|
+
---
|
|
124
|
+
|
|
125
|
+
## Installation
|
|
126
|
+
|
|
127
|
+
```bash
|
|
128
|
+
pip install turbotext
|
|
129
|
+
```
|
|
130
|
+
|
|
131
|
+
**With the Cython extension** *(recommended for exact matching at scale)*:
|
|
132
|
+
|
|
133
|
+
```bash
|
|
134
|
+
pip install cython turbotext
|
|
135
|
+
python setup.py build_ext --inplace
|
|
136
|
+
```
|
|
137
|
+
|
|
138
|
+
The extension is fully optional — TurboText falls back to pure Python automatically when it is not compiled.
|
|
139
|
+
|
|
140
|
+
<details>
|
|
141
|
+
<summary>Install with uv (development)</summary>
|
|
142
|
+
|
|
143
|
+
```bash
|
|
144
|
+
git clone https://github.com/nishankmahore/TurboText
|
|
145
|
+
cd TurboText
|
|
146
|
+
uv sync --group dev
|
|
147
|
+
python setup.py build_ext --inplace
|
|
148
|
+
```
|
|
149
|
+
|
|
150
|
+
</details>
|
|
151
|
+
|
|
152
|
+
---
|
|
153
|
+
|
|
154
|
+
## Quick Start
|
|
155
|
+
|
|
156
|
+
```python
|
|
157
|
+
from turbotext import KeywordStore, MatchPolicy, FuzzyConfig
|
|
158
|
+
|
|
159
|
+
store = KeywordStore(
|
|
160
|
+
policy=MatchPolicy.LEFTMOST_LONGEST,
|
|
161
|
+
fuzzy=FuzzyConfig(max_edit_distance=1),
|
|
162
|
+
)
|
|
163
|
+
|
|
164
|
+
store.add_keywords({
|
|
165
|
+
"aspirin": ["aspirin", "asprin", "aspirin tablet"],
|
|
166
|
+
"ibuprofen": ["ibuprofen", "ibuprofen tablet"],
|
|
167
|
+
})
|
|
168
|
+
|
|
169
|
+
text = "Patient takes asprin and ibuprofen tablet daily"
|
|
170
|
+
|
|
171
|
+
for m in store.extract(text):
|
|
172
|
+
print(f"{m.canonical:12} ed={m.edit_distance} [{m.start}:{m.end}] '{m.text}'")
|
|
173
|
+
# aspirin ed=1 [14:20] 'asprin'
|
|
174
|
+
# ibuprofen ed=0 [25:42] 'ibuprofen tablet'
|
|
175
|
+
|
|
176
|
+
print(store.replace(text))
|
|
177
|
+
# Patient takes aspirin and ibuprofen daily
|
|
178
|
+
```
|
|
179
|
+
|
|
180
|
+
---
|
|
181
|
+
|
|
182
|
+
## Adding Keywords
|
|
183
|
+
|
|
184
|
+
### Single keyword
|
|
185
|
+
|
|
186
|
+
```python
|
|
187
|
+
store = KeywordStore()
|
|
188
|
+
|
|
189
|
+
# minimal — surface form becomes the canonical
|
|
190
|
+
store.add_keyword("aspirin")
|
|
191
|
+
|
|
192
|
+
# with canonical form
|
|
193
|
+
store.add_keyword("aspirin", canonical="Aspirin")
|
|
194
|
+
|
|
195
|
+
# with full metadata
|
|
196
|
+
kid = store.add_keyword(
|
|
197
|
+
"aspirin",
|
|
198
|
+
canonical="Aspirin",
|
|
199
|
+
category="DRUG",
|
|
200
|
+
priority=10.0,
|
|
201
|
+
rxnorm_code="1191", # any extra kwargs become metadata
|
|
202
|
+
source="rxnorm",
|
|
203
|
+
)
|
|
204
|
+
print(kid) # "3f4a2b1c-..." UUID — useful for tracking
|
|
205
|
+
```
|
|
206
|
+
|
|
207
|
+
### From a list
|
|
208
|
+
|
|
209
|
+
```python
|
|
210
|
+
store.add_keywords(["java", "python", "rust"])
|
|
211
|
+
# surface form = canonical for each entry
|
|
212
|
+
```
|
|
213
|
+
|
|
214
|
+
### From a `{surface: canonical}` dict
|
|
215
|
+
|
|
216
|
+
```python
|
|
217
|
+
store.add_keywords({
|
|
218
|
+
"py": "Python",
|
|
219
|
+
"js": "JavaScript",
|
|
220
|
+
"k8s": "Kubernetes",
|
|
221
|
+
})
|
|
222
|
+
```
|
|
223
|
+
|
|
224
|
+
### From a `{canonical: [surfaces]}` dict *(most common bulk shape)*
|
|
225
|
+
|
|
226
|
+
```python
|
|
227
|
+
keyword_dict = {
|
|
228
|
+
"java": ["java", "java_2e", "java programing"],
|
|
229
|
+
"product management": ["PM", "product manager", "prod mgmt"],
|
|
230
|
+
"machine learning": ["ML", "machine learning", "deep learning"],
|
|
231
|
+
}
|
|
232
|
+
store.add_keywords(keyword_dict)
|
|
233
|
+
```
|
|
234
|
+
|
|
235
|
+
### Mixed dict
|
|
236
|
+
|
|
237
|
+
String values follow `{surface: canonical}`; list values follow `{canonical: [surfaces]}`.
|
|
238
|
+
|
|
239
|
+
```python
|
|
240
|
+
store.add_keywords({
|
|
241
|
+
"java": ["java_2e", "java programing"], # canonical → [surfaces]
|
|
242
|
+
"py": "Python", # surface → canonical
|
|
243
|
+
})
|
|
244
|
+
```
|
|
245
|
+
|
|
246
|
+
### Shared category and priority
|
|
247
|
+
|
|
248
|
+
```python
|
|
249
|
+
store.add_keywords(
|
|
250
|
+
{
|
|
251
|
+
"aspirin": ["aspirin", "asprin"],
|
|
252
|
+
"ibuprofen": ["ibuprofen", "advil"],
|
|
253
|
+
},
|
|
254
|
+
category="DRUG",
|
|
255
|
+
priority=5.0,
|
|
256
|
+
)
|
|
257
|
+
```
|
|
258
|
+
|
|
259
|
+
---
|
|
260
|
+
|
|
261
|
+
## Extracting Matches
|
|
262
|
+
|
|
263
|
+
```python
|
|
264
|
+
matches = store.extract("take aspirin and ibuprofen daily")
|
|
265
|
+
|
|
266
|
+
for m in matches:
|
|
267
|
+
print(m.text) # surface text found in the input
|
|
268
|
+
print(m.canonical) # normalised keyword name
|
|
269
|
+
print(m.start, m.end) # character offsets — text[m.start:m.end]
|
|
270
|
+
print(m.edit_distance) # 0 = exact, 1 = one typo, etc.
|
|
271
|
+
print(m.category) # "DRUG"
|
|
272
|
+
print(m.priority) # 10.0
|
|
273
|
+
print(m.keyword_id) # UUID string
|
|
274
|
+
print(m.metadata) # {"rxnorm_code": "1191", ...}
|
|
275
|
+
```
|
|
276
|
+
|
|
277
|
+
`Match` is a **frozen dataclass** — hashable and safe to use as a dict key or set member.
|
|
278
|
+
|
|
279
|
+
---
|
|
280
|
+
|
|
281
|
+
## Replacing Keywords
|
|
282
|
+
|
|
283
|
+
```python
|
|
284
|
+
store = KeywordStore()
|
|
285
|
+
store.add_keywords({
|
|
286
|
+
"aspirin": "Aspirin",
|
|
287
|
+
"tylenol": "Acetaminophen", # alias → same canonical
|
|
288
|
+
"ibuprofen": "Ibuprofen",
|
|
289
|
+
})
|
|
290
|
+
|
|
291
|
+
print(store.replace("take aspirin or tylenol twice daily"))
|
|
292
|
+
# take Aspirin or Acetaminophen twice daily
|
|
293
|
+
```
|
|
294
|
+
|
|
295
|
+
---
|
|
296
|
+
|
|
297
|
+
## Fuzzy Matching
|
|
298
|
+
|
|
299
|
+
```python
|
|
300
|
+
from turbotext import FuzzyConfig
|
|
301
|
+
|
|
302
|
+
store = KeywordStore(fuzzy=FuzzyConfig(max_edit_distance=1))
|
|
303
|
+
store.add_keyword("aspirin")
|
|
304
|
+
|
|
305
|
+
store.extract("take asprin") # substitution i → r ✅
|
|
306
|
+
store.extract("take aspirn") # deletion missing i ✅
|
|
307
|
+
store.extract("take aspirrin") # insertion extra r ✅
|
|
308
|
+
```
|
|
309
|
+
|
|
310
|
+
### Boundary enforcement still applies
|
|
311
|
+
|
|
312
|
+
```python
|
|
313
|
+
store.extract("I see cbt") # ✅ whole word
|
|
314
|
+
store.extract("I see xcbt") # ❌ left boundary fails
|
|
315
|
+
store.extract("I see cbts") # ❌ right boundary fails
|
|
316
|
+
```
|
|
317
|
+
|
|
318
|
+
### Choosing `max_edit_distance`
|
|
319
|
+
|
|
320
|
+
| Value | Use case |
|
|
321
|
+
|:---:|---|
|
|
322
|
+
| `0` | Exact matching only *(default)* — uses Cython fast-path when available |
|
|
323
|
+
| `1` | Single-character typos — medical terms, product names |
|
|
324
|
+
| `2` | Two-character errors — longer technical terms |
|
|
325
|
+
|
|
326
|
+
> Higher values increase recall but also false positives. Start with `1` and tune.
|
|
327
|
+
|
|
328
|
+
---
|
|
329
|
+
|
|
330
|
+
## Conflict Resolution Policies
|
|
331
|
+
|
|
332
|
+
When keywords overlap, the policy decides which match to keep.
|
|
333
|
+
|
|
334
|
+
<details>
|
|
335
|
+
<summary><b>ALL_OVERLAPS</b> — return everything, you decide</summary>
|
|
336
|
+
|
|
337
|
+
```python
|
|
338
|
+
store = KeywordStore(policy=MatchPolicy.ALL_OVERLAPS)
|
|
339
|
+
store.add_keywords(["new", "new york", "york"])
|
|
340
|
+
|
|
341
|
+
store.extract("new york")
|
|
342
|
+
# → ["new", "new york", "york"]
|
|
343
|
+
```
|
|
344
|
+
|
|
345
|
+
</details>
|
|
346
|
+
|
|
347
|
+
<details>
|
|
348
|
+
<summary><b>LEFTMOST_LONGEST</b> (default) — FlashText-compatible greedy</summary>
|
|
349
|
+
|
|
350
|
+
```python
|
|
351
|
+
store = KeywordStore(policy=MatchPolicy.LEFTMOST_LONGEST)
|
|
352
|
+
store.add_keywords(["new", "new york"])
|
|
353
|
+
|
|
354
|
+
store.extract("new york")
|
|
355
|
+
# → ["new york"] longest match wins
|
|
356
|
+
```
|
|
357
|
+
|
|
358
|
+
</details>
|
|
359
|
+
|
|
360
|
+
<details>
|
|
361
|
+
<summary><b>LEFTMOST_FIRST</b> — earliest start wins</summary>
|
|
362
|
+
|
|
363
|
+
```python
|
|
364
|
+
store = KeywordStore(policy=MatchPolicy.LEFTMOST_FIRST)
|
|
365
|
+
store.add_keywords(["new", "new york"])
|
|
366
|
+
|
|
367
|
+
store.extract("new york")
|
|
368
|
+
# → ["new"] first token wins
|
|
369
|
+
```
|
|
370
|
+
|
|
371
|
+
</details>
|
|
372
|
+
|
|
373
|
+
<details>
|
|
374
|
+
<summary><b>HIGHEST_PRIORITY</b> — priority-based greedy</summary>
|
|
375
|
+
|
|
376
|
+
```python
|
|
377
|
+
store = KeywordStore(policy=MatchPolicy.HIGHEST_PRIORITY)
|
|
378
|
+
store.add_keyword("new york", canonical="New York", priority=2.0)
|
|
379
|
+
store.add_keyword("york", canonical="York", priority=5.0)
|
|
380
|
+
store.add_keyword("new", canonical="New", priority=1.0)
|
|
381
|
+
|
|
382
|
+
store.extract("new york")
|
|
383
|
+
# → ["New", "York"] priority 5 beats priority 2; "new" doesn't overlap "york"
|
|
384
|
+
```
|
|
385
|
+
|
|
386
|
+
</details>
|
|
387
|
+
|
|
388
|
+
<details>
|
|
389
|
+
<summary><b>OPTIMAL_WEIGHTED</b> — globally optimal, not greedy</summary>
|
|
390
|
+
|
|
391
|
+
```python
|
|
392
|
+
store = KeywordStore(policy=MatchPolicy.OPTIMAL_WEIGHTED)
|
|
393
|
+
store.add_keyword("ab cd", canonical="LONG", priority=5.0)
|
|
394
|
+
store.add_keyword("ab", canonical="SHORT1", priority=3.0)
|
|
395
|
+
store.add_keyword("cd", canonical="SHORT2", priority=3.0)
|
|
396
|
+
|
|
397
|
+
store.extract("ab cd")
|
|
398
|
+
# Greedy picks "LONG" (5). Optimal picks "SHORT1"+"SHORT2" (3+3=6).
|
|
399
|
+
# → ["SHORT1", "SHORT2"]
|
|
400
|
+
```
|
|
401
|
+
|
|
402
|
+
</details>
|
|
403
|
+
|
|
404
|
+
---
|
|
405
|
+
|
|
406
|
+
## Per-Keyword Metadata
|
|
407
|
+
|
|
408
|
+
```python
|
|
409
|
+
store.add_keyword(
|
|
410
|
+
"aspirin",
|
|
411
|
+
canonical="Aspirin",
|
|
412
|
+
category="DRUG",
|
|
413
|
+
priority=10.0,
|
|
414
|
+
rxnorm_code="1191",
|
|
415
|
+
source="rxnorm",
|
|
416
|
+
approved=True,
|
|
417
|
+
)
|
|
418
|
+
|
|
419
|
+
m = store.extract("take aspirin")[0]
|
|
420
|
+
print(m.canonical) # "Aspirin"
|
|
421
|
+
print(m.category) # "DRUG"
|
|
422
|
+
print(m.priority) # 10.0
|
|
423
|
+
print(m.metadata["rxnorm_code"]) # "1191"
|
|
424
|
+
```
|
|
425
|
+
|
|
426
|
+
`Match.metadata` is a **shallow copy** — mutating it does not affect the stored keyword.
|
|
427
|
+
|
|
428
|
+
---
|
|
429
|
+
|
|
430
|
+
## Word Boundary Rules
|
|
431
|
+
|
|
432
|
+
TurboText uses Unicode word boundaries — word characters are `[a-zA-Z0-9_]` and their Unicode equivalents.
|
|
433
|
+
|
|
434
|
+
```python
|
|
435
|
+
store.add_keyword("cat")
|
|
436
|
+
|
|
437
|
+
# ✅ Accepted
|
|
438
|
+
store.extract("cat") # text edge
|
|
439
|
+
store.extract("the cat sat") # spaces
|
|
440
|
+
store.extract("(cat)") # punctuation
|
|
441
|
+
store.extract("cat, sat") # comma
|
|
442
|
+
|
|
443
|
+
# ❌ Rejected
|
|
444
|
+
store.extract("cats") # right boundary fails
|
|
445
|
+
store.extract("scat") # left boundary fails
|
|
446
|
+
store.extract("cat2") # digit is a word char
|
|
447
|
+
store.extract("concatenate") # substring
|
|
448
|
+
```
|
|
449
|
+
|
|
450
|
+
---
|
|
451
|
+
|
|
452
|
+
## Performance
|
|
453
|
+
|
|
454
|
+
> Apple M-series · 100 → 20,000 keywords · ~20 k-char text · best-of-3 runs
|
|
455
|
+
|
|
456
|
+
### Exact matching (k=0)
|
|
457
|
+
|
|
458
|
+
TurboText + Cython matches FlashText and stays flat as vocabulary grows. `re` degrades because its alternation NFA grows with term count.
|
|
459
|
+
|
|
460
|
+

|
|
461
|
+
|
|
462
|
+
| Library | 100 terms | 5,000 terms | 20,000 terms | Scales? |
|
|
463
|
+
|---|:---:|:---:|:---:|:---:|
|
|
464
|
+
| **TurboText** | 5 ms | 5 ms | 6 ms | O(text) |
|
|
465
|
+
| FlashText | 4 ms | 3 ms | 3 ms | O(text) |
|
|
466
|
+
| `re` | 6 ms | 212 ms | 837 ms | O(text × vocab) |
|
|
467
|
+
|
|
468
|
+
### Fuzzy matching (k=1)
|
|
469
|
+
|
|
470
|
+
TurboText does a **single-pass trie scan** regardless of vocabulary size. RapidFuzz and FuzzyWuzzy tokenise then compare every token against every keyword — O(tokens × vocab).
|
|
471
|
+
|
|
472
|
+

|
|
473
|
+
|
|
474
|
+
| Library | 100 terms | 1,000 terms | 5,000 terms | Boundary-aware | Extracts spans |
|
|
475
|
+
|---|:---:|:---:|:---:|:---:|:---:|
|
|
476
|
+
| **TurboText** | 96 ms | 174 ms | 239 ms | ✅ | ✅ |
|
|
477
|
+
| RapidFuzz | 13 ms | 120 ms | 597 ms | ❌ | ❌ |
|
|
478
|
+
| FuzzyWuzzy | 222 ms | 2,205 ms | 10,917 ms | ❌ | ❌ |
|
|
479
|
+
|
|
480
|
+
> **Note:** RapidFuzz is a string-similarity scorer, not a text scanner — benchmarks use `process.extractOne` per token. It does not return character offsets or enforce word boundaries. TurboText does a single trie pass over the full document and natively supports multi-word phrases.
|
|
481
|
+
|
|
482
|
+
TurboText crosses RapidFuzz at ~2,000 keywords. A Cython extension for the fuzzy frontier is planned for v2.
|
|
483
|
+
|
|
484
|
+
---
|
|
485
|
+
|
|
486
|
+
## API Reference
|
|
487
|
+
|
|
488
|
+
<details>
|
|
489
|
+
<summary><b>KeywordStore(policy, fuzzy)</b></summary>
|
|
490
|
+
|
|
491
|
+
```python
|
|
492
|
+
store = KeywordStore(
|
|
493
|
+
policy=MatchPolicy.LEFTMOST_LONGEST, # default
|
|
494
|
+
fuzzy=FuzzyConfig(max_edit_distance=0), # default — exact only
|
|
495
|
+
)
|
|
496
|
+
```
|
|
497
|
+
|
|
498
|
+
| Parameter | Type | Default | Description |
|
|
499
|
+
|---|---|---|---|
|
|
500
|
+
| `policy` | `MatchPolicy` | `LEFTMOST_LONGEST` | Conflict-resolution strategy |
|
|
501
|
+
| `fuzzy` | `FuzzyConfig \| None` | `None` | Fuzzy config; `None` = exact only |
|
|
502
|
+
|
|
503
|
+
</details>
|
|
504
|
+
|
|
505
|
+
<details>
|
|
506
|
+
<summary><b>add_keyword(surface_form, *, canonical, category, priority, **metadata) → str</b></summary>
|
|
507
|
+
|
|
508
|
+
```python
|
|
509
|
+
kid = store.add_keyword(
|
|
510
|
+
"aspirin",
|
|
511
|
+
canonical="Aspirin",
|
|
512
|
+
category="DRUG",
|
|
513
|
+
priority=10.0,
|
|
514
|
+
rxnorm_code="1191",
|
|
515
|
+
)
|
|
516
|
+
```
|
|
517
|
+
|
|
518
|
+
| Parameter | Type | Default | Description |
|
|
519
|
+
|---|---|---|---|
|
|
520
|
+
| `surface_form` | `str` | required | Text to search for |
|
|
521
|
+
| `canonical` | `str` | `surface_form` | Normalised name returned on match |
|
|
522
|
+
| `category` | `str \| None` | `None` | Grouping label |
|
|
523
|
+
| `priority` | `float` | `1.0` | Weight for priority-based policies |
|
|
524
|
+
| `**metadata` | `Any` | — | Arbitrary extra fields |
|
|
525
|
+
|
|
526
|
+
Returns the keyword's UUID string.
|
|
527
|
+
|
|
528
|
+
</details>
|
|
529
|
+
|
|
530
|
+
<details>
|
|
531
|
+
<summary><b>add_keywords(keywords, *, category, priority) → list[str]</b></summary>
|
|
532
|
+
|
|
533
|
+
| Shape | Example |
|
|
534
|
+
|---|---|
|
|
535
|
+
| `list[str]` | `["java", "python"]` |
|
|
536
|
+
| `dict[str, str]` | `{"py": "Python"}` — surface → canonical |
|
|
537
|
+
| `dict[str, list[str]]` | `{"Python": ["py", "python3"]}` — canonical → surfaces |
|
|
538
|
+
|
|
539
|
+
`category` and `priority` apply to every keyword in the call.
|
|
540
|
+
|
|
541
|
+
</details>
|
|
542
|
+
|
|
543
|
+
<details>
|
|
544
|
+
<summary><b>extract(text) → list[Match]</b></summary>
|
|
545
|
+
|
|
546
|
+
Returns resolved matches in span order.
|
|
547
|
+
|
|
548
|
+
```python
|
|
549
|
+
matches = store.extract("Patient takes aspirin daily")
|
|
550
|
+
```
|
|
551
|
+
|
|
552
|
+
</details>
|
|
553
|
+
|
|
554
|
+
<details>
|
|
555
|
+
<summary><b>replace(text) → str</b></summary>
|
|
556
|
+
|
|
557
|
+
Replaces every matched span with its canonical form.
|
|
558
|
+
|
|
559
|
+
```python
|
|
560
|
+
result = store.replace("take aspirin or tylenol")
|
|
561
|
+
```
|
|
562
|
+
|
|
563
|
+
</details>
|
|
564
|
+
|
|
565
|
+
<details>
|
|
566
|
+
<summary><b>Match fields</b></summary>
|
|
567
|
+
|
|
568
|
+
`Match` is a frozen dataclass — all fields are read-only and it is hashable.
|
|
569
|
+
|
|
570
|
+
| Field | Type | Description |
|
|
571
|
+
|---|---|---|
|
|
572
|
+
| `text` | `str` | Matched surface text as it appears in the input |
|
|
573
|
+
| `canonical` | `str` | Normalised form from `add_keyword` |
|
|
574
|
+
| `start` | `int` | Start character offset |
|
|
575
|
+
| `end` | `int` | End character offset (exclusive) |
|
|
576
|
+
| `edit_distance` | `int` | Levenshtein distance (0 = exact) |
|
|
577
|
+
| `category` | `str \| None` | User-supplied category |
|
|
578
|
+
| `priority` | `float` | User-supplied priority |
|
|
579
|
+
| `keyword_id` | `str` | UUID from `add_keyword` |
|
|
580
|
+
| `metadata` | `dict` | Copy of extra kwargs |
|
|
581
|
+
|
|
582
|
+
</details>
|
|
583
|
+
|
|
584
|
+
<details>
|
|
585
|
+
<summary><b>FuzzyConfig and MatchPolicy</b></summary>
|
|
586
|
+
|
|
587
|
+
```python
|
|
588
|
+
FuzzyConfig(max_edit_distance=1) # int, default 0
|
|
589
|
+
```
|
|
590
|
+
|
|
591
|
+
| `MatchPolicy` | Description |
|
|
592
|
+
|---|---|
|
|
593
|
+
| `ALL_OVERLAPS` | Return every match |
|
|
594
|
+
| `LEFTMOST_LONGEST` | Greedy — leftmost, then longest |
|
|
595
|
+
| `LEFTMOST_FIRST` | Greedy — leftmost, then insertion order |
|
|
596
|
+
| `HIGHEST_PRIORITY` | Greedy — highest priority wins cluster |
|
|
597
|
+
| `OPTIMAL_WEIGHTED` | Exact — maximise total priority globally |
|
|
598
|
+
|
|
599
|
+
</details>
|
|
600
|
+
|
|
601
|
+
---
|
|
602
|
+
|
|
603
|
+
## Development
|
|
604
|
+
|
|
605
|
+
```bash
|
|
606
|
+
uv sync --group dev # install dependencies
|
|
607
|
+
python setup.py build_ext --inplace # build Cython extension (optional)
|
|
608
|
+
uv run pytest # run tests
|
|
609
|
+
uv run ruff check src tests # lint
|
|
610
|
+
uv run mypy src/turbotext # type-check
|
|
611
|
+
uv run pytest benches/ --benchmark-only # throughput benchmarks
|
|
612
|
+
uv run python benches/scaling_benchmark.py # regenerate scaling charts
|
|
613
|
+
```
|
|
614
|
+
|
|
615
|
+
<details>
|
|
616
|
+
<summary>Project layout</summary>
|
|
617
|
+
|
|
618
|
+
```
|
|
619
|
+
src/turbotext/
|
|
620
|
+
__init__.py public API exports
|
|
621
|
+
trie.py TrieNode + TrieBuilder
|
|
622
|
+
frontier.py bounded-edit frontier search (pure Python)
|
|
623
|
+
_fast.pyx Cython hot-path for exact search (k=0)
|
|
624
|
+
_fast.pyi type stub for the Cython extension
|
|
625
|
+
resolve.py conflict-resolution policies
|
|
626
|
+
store.py KeywordStore public class
|
|
627
|
+
reference/
|
|
628
|
+
reference_matcher.py brute-force oracle for differential testing
|
|
629
|
+
tests/
|
|
630
|
+
test_m0_smoke.py API surface + add_keywords shapes
|
|
631
|
+
test_m1_exact.py exact matching + boundary rules
|
|
632
|
+
test_m2_metadata_priorities.py metadata, HIGHEST_PRIORITY, OPTIMAL_WEIGHTED
|
|
633
|
+
test_m3_fuzzy.py fuzzy matching + hypothesis property tests
|
|
634
|
+
benches/
|
|
635
|
+
bench_m3.py pytest-benchmark: k=0 vs k=1 throughput
|
|
636
|
+
bench_comparison.py pytest-benchmark: TurboText vs FlashText vs re vs RapidFuzz
|
|
637
|
+
scaling_benchmark.py vocabulary-sweep scaling charts
|
|
638
|
+
assets/
|
|
639
|
+
logo.png
|
|
640
|
+
```
|
|
641
|
+
|
|
642
|
+
</details>
|
|
643
|
+
|
|
644
|
+
---
|
|
645
|
+
|
|
646
|
+
## Author
|
|
647
|
+
|
|
648
|
+
<p>
|
|
649
|
+
<b>Nishank Mahore</b><br/>
|
|
650
|
+
<a href="mailto:nishankmahore@gmail.com">nishankmahore@gmail.com</a> ·
|
|
651
|
+
<a href="https://github.com/nishankmahore">github.com/nishankmahore</a>
|
|
652
|
+
</p>
|
|
653
|
+
|
|
654
|
+
If TurboText is useful to you, feel free to open an issue, suggest a feature, or contribute a pull request.
|
|
655
|
+
|
|
656
|
+
---
|
|
657
|
+
|
|
658
|
+
## License
|
|
659
|
+
|
|
660
|
+
Released under the **MIT License** — see [LICENSE](LICENSE) for the full text.
|
|
661
|
+
|
|
662
|
+
---
|
|
663
|
+
|
|
664
|
+
<p align="center">
|
|
665
|
+
Made with Python · Powered by Cython · Built for speed
|
|
666
|
+
</p>
|