structflo-ner 0.2.1__tar.gz → 0.2.2__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (36) hide show
  1. {structflo_ner-0.2.1 → structflo_ner-0.2.2}/.github/workflows/ci.yml +1 -1
  2. {structflo_ner-0.2.1 → structflo_ner-0.2.2}/PKG-INFO +4 -1
  3. structflo_ner-0.2.2/notebooks/02_fast_ner.ipynb +416 -0
  4. {structflo_ner-0.2.1 → structflo_ner-0.2.2}/pyproject.toml +1 -0
  5. {structflo_ner-0.2.1 → structflo_ner-0.2.2}/structflo/ner/__init__.py +9 -2
  6. {structflo_ner-0.2.1 → structflo_ner-0.2.2}/structflo/ner/extractor.py +1 -2
  7. structflo_ner-0.2.2/structflo/ner/fast/README.md +149 -0
  8. structflo_ner-0.2.2/structflo/ner/fast/__init__.py +22 -0
  9. structflo_ner-0.2.2/structflo/ner/fast/_loader.py +116 -0
  10. structflo_ner-0.2.2/structflo/ner/fast/_matcher.py +280 -0
  11. structflo_ner-0.2.2/structflo/ner/fast/_normalize.py +84 -0
  12. structflo_ner-0.2.2/structflo/ner/fast/extractor.py +160 -0
  13. structflo_ner-0.2.2/structflo/ner/fast/gazetteers/accession_number.yml +18 -0
  14. structflo_ner-0.2.2/structflo/ner/fast/gazetteers/compound_name.yml +50 -0
  15. structflo_ner-0.2.2/structflo/ner/fast/gazetteers/disease.yml +22 -0
  16. structflo_ner-0.2.2/structflo/ner/fast/gazetteers/functional_category.yml +27 -0
  17. structflo_ner-0.2.2/structflo/ner/fast/gazetteers/gene_name.yml +75 -0
  18. structflo_ner-0.2.2/structflo/ner/fast/gazetteers/product.yml +34 -0
  19. structflo_ner-0.2.2/structflo/ner/fast/gazetteers/screening_method.yml +38 -0
  20. structflo_ner-0.2.2/structflo/ner/fast/gazetteers/target.yml +78 -0
  21. {structflo_ner-0.2.1 → structflo_ner-0.2.2}/tests/test_extractor.py +2 -6
  22. structflo_ner-0.2.2/tests/test_fast.py +377 -0
  23. {structflo_ner-0.2.1 → structflo_ner-0.2.2}/uv.lock +97 -1
  24. {structflo_ner-0.2.1 → structflo_ner-0.2.2}/.github/workflows/publish.yml +0 -0
  25. {structflo_ner-0.2.1 → structflo_ner-0.2.2}/.gitignore +0 -0
  26. {structflo_ner-0.2.1 → structflo_ner-0.2.2}/Makefile +0 -0
  27. {structflo_ner-0.2.1 → structflo_ner-0.2.2}/README.md +0 -0
  28. {structflo_ner-0.2.1 → structflo_ner-0.2.2}/notebooks/01_quickstart.ipynb +0 -0
  29. {structflo_ner-0.2.1 → structflo_ner-0.2.2}/structflo/ner/_display.py +0 -0
  30. {structflo_ner-0.2.1 → structflo_ner-0.2.2}/structflo/ner/_entities.py +0 -0
  31. {structflo_ner-0.2.1 → structflo_ner-0.2.2}/structflo/ner/_examples.py +0 -0
  32. {structflo_ner-0.2.1 → structflo_ner-0.2.2}/structflo/ner/_mapping.py +0 -0
  33. {structflo_ner-0.2.1 → structflo_ner-0.2.2}/structflo/ner/_prompts.py +0 -0
  34. {structflo_ner-0.2.1 → structflo_ner-0.2.2}/structflo/ner/profiles.py +0 -0
  35. {structflo_ner-0.2.1 → structflo_ner-0.2.2}/tests/__init__.py +0 -0
  36. {structflo_ner-0.2.1 → structflo_ner-0.2.2}/tests/test_entities.py +0 -0
@@ -19,7 +19,7 @@ jobs:
19
19
  run: uv python install 3.12
20
20
 
21
21
  - name: Install dependencies
22
- run: uv sync --dev
22
+ run: uv sync --dev --extra fast --extra dataframe
23
23
 
24
24
  - name: Lint
25
25
  run: uv run ruff check structflo/ tests/
@@ -1,12 +1,15 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: structflo-ner
3
- Version: 0.2.1
3
+ Version: 0.2.2
4
4
  Summary: Drug discovery NER wrapper around LangExtract — zero-config entity extraction for chemistry and biology.
5
5
  License: Apache-2.0
6
6
  Requires-Python: >=3.10
7
7
  Requires-Dist: langextract>=1.1.1
8
8
  Provides-Extra: dataframe
9
9
  Requires-Dist: pandas>=1.5; extra == 'dataframe'
10
+ Provides-Extra: fast
11
+ Requires-Dist: pyyaml>=6.0; extra == 'fast'
12
+ Requires-Dist: rapidfuzz>=3.0; extra == 'fast'
10
13
  Description-Content-Type: text/markdown
11
14
 
12
15
  # structflo.ner
@@ -0,0 +1,416 @@
1
+ {
2
+ "cells": [
3
+ {
4
+ "cell_type": "markdown",
5
+ "id": "cell-0",
6
+ "metadata": {},
7
+ "source": [
8
+ "# Fast Dictionary-Based NER for TB Drug Discovery\n",
9
+ "\n",
10
+ "This notebook demonstrates `FastNERExtractor` — a fast, deterministic NER\n",
11
+ "engine that uses curated YAML gazetteers instead of an LLM.\n",
12
+ "\n",
13
+ "1. Basic usage\n",
14
+ "2. How matching works (exact, regex, fuzzy)\n",
15
+ "3. Working with results (same as LLM extractor)\n",
16
+ "4. Custom gazetteers\n",
17
+ "5. Adding new entity types\n",
18
+ "6. Batch extraction & performance"
19
+ ]
20
+ },
21
+ {
22
+ "cell_type": "markdown",
23
+ "id": "cell-1",
24
+ "metadata": {},
25
+ "source": [
26
+ "## Setup\n",
27
+ "\n",
28
+ "```bash\n",
29
+ "uv add \"structflo-ner[fast]\"\n",
30
+ "# for DataFrame support\n",
31
+ "uv add \"structflo-ner[fast,dataframe]\"\n",
32
+ "```"
33
+ ]
34
+ },
35
+ {
36
+ "cell_type": "code",
37
+ "execution_count": null,
38
+ "id": "cell-2",
39
+ "metadata": {},
40
+ "outputs": [],
41
+ "source": [
42
+ "from structflo.ner.fast import FastNERExtractor"
43
+ ]
44
+ },
45
+ {
46
+ "cell_type": "markdown",
47
+ "id": "cell-3",
48
+ "metadata": {},
49
+ "source": [
50
+ "## 1. Basic Usage\n",
51
+ "\n",
52
+ "No API key, no LLM, no network — just instantiate and extract.\n",
53
+ "The built-in gazetteers cover TB drug discovery entities."
54
+ ]
55
+ },
56
+ {
57
+ "cell_type": "code",
58
+ "execution_count": null,
59
+ "id": "cell-4",
60
+ "metadata": {},
61
+ "outputs": [],
62
+ "source": [
63
+ "fast = FastNERExtractor()\n",
64
+ "\n",
65
+ "text = (\n",
66
+ " \"Bedaquiline (TMC207) is a diarylquinoline that inhibits the \"\n",
67
+ " \"mycobacterial ATP synthase subunit c encoded by atpE (Rv1305). \"\n",
68
+ " \"It shows potent activity against Mycobacterium tuberculosis \"\n",
69
+ " \"including MDR-TB and XDR-TB. The compound was identified through \"\n",
70
+ " \"whole-cell screening and targets the energy metabolism pathway.\"\n",
71
+ ")\n",
72
+ "\n",
73
+ "result = fast.extract(text)\n",
74
+ "result"
75
+ ]
76
+ },
77
+ {
78
+ "cell_type": "markdown",
79
+ "id": "cell-5",
80
+ "metadata": {},
81
+ "source": [
82
+ "## 2. How Matching Works\n",
83
+ "\n",
84
+ "The extractor uses a three-phase matching strategy:\n",
85
+ "\n",
86
+ "| Phase | Method | What it catches |\n",
87
+ "|---|---|---|\n",
88
+ "| 1 | **Exact match** | Case-sensitive and normalized dictionary lookups with word-boundary enforcement |\n",
89
+ "| 1b | **Regex patterns** | Auto-derived from accession number seeds (Rv tags, UniProt, PDB, etc.) |\n",
90
+ "| 2 | **Fuzzy match** | Typos and minor variants via rapidfuzz (configurable threshold) |\n",
91
+ "\n",
92
+ "Each entity's `attributes` dict includes the `match_method` used."
93
+ ]
94
+ },
95
+ {
96
+ "cell_type": "code",
97
+ "execution_count": null,
98
+ "id": "cell-6",
99
+ "metadata": {},
100
+ "outputs": [],
101
+ "source": [
102
+ "# Inspect match methods\n",
103
+ "for entity in result.all_entities():\n",
104
+ " method = entity.attributes.get(\"match_method\", \"\")\n",
105
+ " canonical = entity.attributes.get(\"canonical\", entity.text)\n",
106
+ " print(f\"{entity.entity_type:25s} | {entity.text:30s} | method={method:6s} | canonical={canonical}\")"
107
+ ]
108
+ },
109
+ {
110
+ "cell_type": "markdown",
111
+ "id": "cell-7",
112
+ "metadata": {},
113
+ "source": [
114
+ "### Regex matching for accession numbers\n",
115
+ "\n",
116
+ "Seed entries in `accession_number.yml` auto-derive regex patterns.\n",
117
+ "For example, `Rv0005` teaches the system `Rv\\d{4}[c]?`, so *all* Rv locus tags\n",
118
+ "are matched — not just the ones listed."
119
+ ]
120
+ },
121
+ {
122
+ "cell_type": "code",
123
+ "execution_count": null,
124
+ "id": "cell-8",
125
+ "metadata": {},
126
+ "outputs": [],
127
+ "source": [
128
+ "# Rv2043c is NOT in the YAML, but the regex pattern catches it\n",
129
+ "result2 = fast.extract(\"PptT is encoded by Rv2043c and is essential for mycolic acid biosynthesis.\")\n",
130
+ "\n",
131
+ "print(\"Accessions found:\")\n",
132
+ "for a in result2.accessions:\n",
133
+ " print(f\" {a.text} (method: {a.attributes['match_method']})\")\n",
134
+ "\n",
135
+ "print(\"\\nTargets found:\")\n",
136
+ "for t in result2.targets:\n",
137
+ " print(f\" {t.text}\")"
138
+ ]
139
+ },
140
+ {
141
+ "cell_type": "markdown",
142
+ "id": "cell-9",
143
+ "metadata": {},
144
+ "source": [
145
+ "### Fuzzy matching\n",
146
+ "\n",
147
+ "Catches typos and minor spelling variants. The threshold (default 85) controls sensitivity."
148
+ ]
149
+ },
150
+ {
151
+ "cell_type": "code",
152
+ "execution_count": null,
153
+ "id": "cell-10",
154
+ "metadata": {},
155
+ "outputs": [],
156
+ "source": [
157
+ "# \"Bedaquilne\" is a typo for \"Bedaquiline\"\n",
158
+ "fuzzy_result = fast.extract(\"Bedaquilne showed activity against TB\")\n",
159
+ "\n",
160
+ "for c in fuzzy_result.compounds:\n",
161
+ " print(f\"Found: {c.text!r} -> canonical: {c.attributes.get('canonical', c.text)!r} (method: {c.attributes['match_method']})\")"
162
+ ]
163
+ },
164
+ {
165
+ "cell_type": "code",
166
+ "execution_count": null,
167
+ "id": "cell-11",
168
+ "metadata": {},
169
+ "outputs": [],
170
+ "source": [
171
+ "# Disable fuzzy matching for strict mode\n",
172
+ "strict = FastNERExtractor(fuzzy_threshold=0)\n",
173
+ "strict_result = strict.extract(\"Bedaquilne showed activity against TB\")\n",
174
+ "\n",
175
+ "print(f\"Compounds (strict): {strict_result.compounds}\") # empty — typo not matched\n",
176
+ "print(f\"Diseases (strict): {[d.text for d in strict_result.diseases]}\") # TB still matched"
177
+ ]
178
+ },
179
+ {
180
+ "cell_type": "markdown",
181
+ "id": "cell-12",
182
+ "metadata": {},
183
+ "source": [
184
+ "## 3. Working with Results\n",
185
+ "\n",
186
+ "`FastNERExtractor` returns the same `NERResult` objects as the LLM-based `NERExtractor`.\n",
187
+ "All downstream tooling works identically."
188
+ ]
189
+ },
190
+ {
191
+ "cell_type": "code",
192
+ "execution_count": null,
193
+ "id": "cell-13",
194
+ "metadata": {},
195
+ "outputs": [],
196
+ "source": [
197
+ "# Typed entity lists\n",
198
+ "print(\"Compounds:\", [c.text for c in result.compounds])\n",
199
+ "print(\"Targets:\", [t.text for t in result.targets])\n",
200
+ "print(\"Diseases:\", [d.text for d in result.diseases])\n",
201
+ "print(\"Accessions:\", [a.text for a in result.accessions])\n",
202
+ "print(\"Screening methods:\", [s.text for s in result.screening_methods])\n",
203
+ "print(\"Products:\", [p.text for p in result.products])\n",
204
+ "print(\"Functional categories:\", [f.text for f in result.functional_categories])"
205
+ ]
206
+ },
207
+ {
208
+ "cell_type": "code",
209
+ "execution_count": null,
210
+ "id": "cell-14",
211
+ "metadata": {},
212
+ "outputs": [],
213
+ "source": [
214
+ "# Export to DataFrame\n",
215
+ "df = result.to_dataframe()\n",
216
+ "df"
217
+ ]
218
+ },
219
+ {
220
+ "cell_type": "code",
221
+ "execution_count": null,
222
+ "id": "cell-15",
223
+ "metadata": {},
224
+ "outputs": [],
225
+ "source": [
226
+ "# Character offsets let you highlight entities in the source text\n",
227
+ "for entity in result.all_entities():\n",
228
+ " if entity.char_start is not None:\n",
229
+ " span = text[entity.char_start:entity.char_end]\n",
230
+ " print(f\"[{entity.char_start:3d}:{entity.char_end:3d}] {entity.entity_type:25s} | {span!r}\")"
231
+ ]
232
+ },
233
+ {
234
+ "cell_type": "markdown",
235
+ "id": "cell-16",
236
+ "metadata": {},
237
+ "source": [
238
+ "## 4. Custom Gazetteers\n",
239
+ "\n",
240
+ "YAML files contain only names — one per line. The filename becomes the `entity_type`.\n",
241
+ "\n",
242
+ "```yaml\n",
243
+ "# my_gazetteers/target.yml\n",
244
+ "- MyNovelTarget\n",
245
+ "- AnotherTarget\n",
246
+ "- KinaseX\n",
247
+ "```\n",
248
+ "\n",
249
+ "```python\n",
250
+ "fast = FastNERExtractor(gazetteer_dir=\"my_gazetteers/\")\n",
251
+ "```\n",
252
+ "\n",
253
+ "You can also add terms programmatically without creating files:"
254
+ ]
255
+ },
256
+ {
257
+ "cell_type": "code",
258
+ "execution_count": null,
259
+ "id": "cell-17",
260
+ "metadata": {},
261
+ "outputs": [],
262
+ "source": [
263
+ "# Add extra terms on top of the built-in gazetteers\n",
264
+ "custom = FastNERExtractor(\n",
265
+ " extra_gazetteers={\n",
266
+ " \"target\": [\"MyNovelTarget\", \"KinaseX\"],\n",
267
+ " \"compound_name\": [\"CompoundABC\"],\n",
268
+ " }\n",
269
+ ")\n",
270
+ "\n",
271
+ "r = custom.extract(\"CompoundABC inhibits MyNovelTarget in M. tuberculosis\")\n",
272
+ "print(\"Compounds:\", [c.text for c in r.compounds])\n",
273
+ "print(\"Targets:\", [t.text for t in r.targets])\n",
274
+ "print(\"Diseases:\", [d.text for d in r.diseases])"
275
+ ]
276
+ },
277
+ {
278
+ "cell_type": "markdown",
279
+ "id": "cell-18",
280
+ "metadata": {},
281
+ "source": [
282
+ "## 5. Adding New Entity Types\n",
283
+ "\n",
284
+ "To add a new gazetteer, just drop a YAML file into the gazetteers directory.\n",
285
+ "The filename (without `.yml`) must match an `entity_type` from the entity class map.\n",
286
+ "\n",
287
+ "**Built-in entity types:**\n",
288
+ "\n",
289
+ "| Filename | entity_type | Python class |\n",
290
+ "|---|---|---|\n",
291
+ "| `target.yml` | target | `TargetEntity` |\n",
292
+ "| `gene_name.yml` | gene_name | `TargetEntity` |\n",
293
+ "| `compound_name.yml` | compound_name | `ChemicalEntity` |\n",
294
+ "| `disease.yml` | disease | `DiseaseEntity` |\n",
295
+ "| `accession_number.yml` | accession_number | `AccessionEntity` |\n",
296
+ "| `screening_method.yml` | screening_method | `ScreeningMethodEntity` |\n",
297
+ "| `functional_category.yml` | functional_category | `FunctionalCategoryEntity` |\n",
298
+ "| `product.yml` | product | `ProductEntity` |\n",
299
+ "\n",
300
+ "**What's auto-derived from names:**\n",
301
+ "- Case variants (InhA, inha, INHA)\n",
302
+ "- Hyphen-optional forms (DprE-1 ↔ DprE1)\n",
303
+ "- Period-optional forms (M. tuberculosis ↔ M tuberculosis)\n",
304
+ "- Greek letter expansion (β-lactam ↔ beta-lactam)\n",
305
+ "- Regex patterns for accession number seeds (Rv, MT, UniProt, PDB, RefSeq)"
306
+ ]
307
+ },
308
+ {
309
+ "cell_type": "code",
310
+ "execution_count": null,
311
+ "id": "cell-19",
312
+ "metadata": {},
313
+ "outputs": [],
314
+ "source": [
315
+ "# See what gazetteers are loaded by default\n",
316
+ "from structflo.ner.fast._loader import load_all_gazetteers\n",
317
+ "\n",
318
+ "gazetteers = load_all_gazetteers()\n",
319
+ "for entity_type, terms in gazetteers.items():\n",
320
+ " print(f\"{entity_type:25s} | {len(terms):3d} terms | first 5: {terms[:5]}\")"
321
+ ]
322
+ },
323
+ {
324
+ "cell_type": "markdown",
325
+ "id": "cell-20",
326
+ "metadata": {},
327
+ "source": [
328
+ "## 6. Batch Extraction & Performance\n",
329
+ "\n",
330
+ "The fast extractor processes text in milliseconds — orders of magnitude faster than LLM-based extraction."
331
+ ]
332
+ },
333
+ {
334
+ "cell_type": "code",
335
+ "execution_count": null,
336
+ "id": "cell-21",
337
+ "metadata": {},
338
+ "outputs": [],
339
+ "source": [
340
+ "abstracts = [\n",
341
+ " \"Bedaquiline inhibits AtpE (Rv1305) with nanomolar potency against MDR-TB.\",\n",
342
+ " \"Delamanid (OPC-67683) is activated by Ddn and targets mycolic acid biosynthesis in M. tuberculosis.\",\n",
343
+ " \"Pretomanid (PA-824) requires activation by Ddn (Rv3547) and kills both replicating and non-replicating Mtb.\",\n",
344
+ " \"PBTZ169 (Macozinone) inhibits DprE1 (Rv3790), an essential enzyme in cell wall biosynthesis.\",\n",
345
+ " \"SQ109 targets MmpL3, a trehalose monomycolate transporter essential for cell wall assembly.\",\n",
346
+ " \"Fragment-based screening identified InhA inhibitors that bypass katG-mediated activation.\",\n",
347
+ " \"CRISPRi screening revealed QcrB (Rv2196) as a vulnerable target in energy metabolism.\",\n",
348
+ " \"Structure-based drug design targeting KasA (Rv2245) yielded novel fatty acid biosynthesis inhibitors.\",\n",
349
+ "]\n",
350
+ "\n",
351
+ "results = fast.extract(abstracts)\n",
352
+ "\n",
353
+ "for i, r in enumerate(results):\n",
354
+ " entities = r.all_entities()\n",
355
+ " print(f\"Abstract {i+1}: {len(entities)} entities — {[e.text for e in entities]}\")"
356
+ ]
357
+ },
358
+ {
359
+ "cell_type": "code",
360
+ "execution_count": null,
361
+ "id": "cell-22",
362
+ "metadata": {},
363
+ "outputs": [],
364
+ "source": [
365
+ "%%timeit -n 100\n",
366
+ "# Benchmark: extract from a single abstract\n",
367
+ "fast.extract(text)"
368
+ ]
369
+ },
370
+ {
371
+ "cell_type": "code",
372
+ "execution_count": null,
373
+ "id": "cell-23",
374
+ "metadata": {},
375
+ "outputs": [],
376
+ "source": [
377
+ "%%timeit -n 10\n",
378
+ "# Benchmark: extract from 8 abstracts\n",
379
+ "fast.extract(abstracts)"
380
+ ]
381
+ },
382
+ {
383
+ "cell_type": "markdown",
384
+ "id": "cell-24",
385
+ "metadata": {},
386
+ "source": [
387
+ "## Comparing Fast vs LLM Extraction\n",
388
+ "\n",
389
+ "The fast extractor is ideal as a **first pass** for bulk screening.\n",
390
+ "Use the LLM extractor for deeper analysis where context and novel entities matter.\n",
391
+ "\n",
392
+ "| | `FastNERExtractor` | `NERExtractor` |\n",
393
+ "|---|---|---|\n",
394
+ "| Speed | ~1-5ms per abstract | ~2-5s per abstract |\n",
395
+ "| Novel entities | Only known terms | Discovers new entities |\n",
396
+ "| Context awareness | None (string matching) | Full contextual understanding |\n",
397
+ "| Cost | Free (no API calls) | API costs or GPU |\n",
398
+ "| Setup | Zero config | API key or Ollama |\n",
399
+ "| Output | `NERResult` | `NERResult` (identical) |"
400
+ ]
401
+ }
402
+ ],
403
+ "metadata": {
404
+ "kernelspec": {
405
+ "display_name": "Python 3",
406
+ "language": "python",
407
+ "name": "python3"
408
+ },
409
+ "language_info": {
410
+ "name": "python",
411
+ "version": "3.12.0"
412
+ }
413
+ },
414
+ "nbformat": 4,
415
+ "nbformat_minor": 5
416
+ }
@@ -11,6 +11,7 @@ dependencies = [
11
11
 
12
12
  [project.optional-dependencies]
13
13
  dataframe = ["pandas>=1.5"]
14
+ fast = ["rapidfuzz>=3.0", "PyYAML>=6.0"]
14
15
 
15
16
  [dependency-groups]
16
17
  dev = [
@@ -49,6 +49,12 @@ from structflo.ner._entities import (
49
49
  TargetEntity,
50
50
  )
51
51
  from structflo.ner.extractor import NERExtractor
52
+
53
+ try:
54
+ from structflo.ner.fast import FastNERExtractor
55
+ except ImportError: # rapidfuzz / PyYAML not installed
56
+ FastNERExtractor = None # type: ignore[assignment,misc]
57
+
52
58
  from structflo.ner.profiles import (
53
59
  BIOACTIVITY,
54
60
  BIOLOGY,
@@ -61,11 +67,12 @@ from structflo.ner.profiles import (
61
67
  EntityProfile,
62
68
  )
63
69
 
64
- __version__ = "0.2.1"
70
+ __version__ = "0.2.2"
65
71
 
66
72
  __all__ = [
67
- # Main class
73
+ # Main classes
68
74
  "NERExtractor",
75
+ "FastNERExtractor",
69
76
  # Profile system
70
77
  "EntityProfile",
71
78
  "FULL",
@@ -146,8 +146,7 @@ class NERExtractor:
146
146
  kept.append(ext)
147
147
  else:
148
148
  logger.warning(
149
- "Dropping extraction with unknown class %r (text=%r). "
150
- "Allowed classes: %s",
149
+ "Dropping extraction with unknown class %r (text=%r). Allowed classes: %s",
151
150
  ext.extraction_class,
152
151
  ext.extraction_text,
153
152
  ", ".join(sorted(allowed_classes)),
@@ -0,0 +1,149 @@
1
+ # structflo.ner.fast — Dictionary-Based NER for TB Drug Discovery
2
+
3
+ Fast, deterministic entity extraction using curated YAML gazetteers. No LLM, no API key, no network — runs in milliseconds.
4
+
5
+ ## Install
6
+
7
+ ```bash
8
+ uv add "structflo-ner[fast]"
9
+
10
+ # with DataFrame support
11
+ uv add "structflo-ner[fast,dataframe]"
12
+ ```
13
+
14
+ ## Quick Start
15
+
16
+ ```python
17
+ from structflo.ner.fast import FastNERExtractor
18
+
19
+ fast = FastNERExtractor()
20
+ result = fast.extract("Bedaquiline inhibits AtpE (Rv1305) in MDR-TB.")
21
+
22
+ print(result.compounds) # [ChemicalEntity(text='Bedaquiline', ...)]
23
+ print(result.targets) # [TargetEntity(text='AtpE', ...)]
24
+ print(result.accessions) # [AccessionEntity(text='Rv1305', ...)]
25
+ print(result.diseases) # [DiseaseEntity(text='MDR-TB', ...)]
26
+
27
+ df = result.to_dataframe()
28
+ result.display() # interactive HTML in Jupyter
29
+ ```
30
+
31
+ ## How It Works
32
+
33
+ Three-phase matching, all without an LLM:
34
+
35
+ ### Phase 1 — Exact Dictionary Match
36
+ Looks up every text span against a normalized dictionary built from the YAML gazetteers. Auto-derived variants include:
37
+ - **Case variants**: InhA, inha, INHA
38
+ - **Hyphen-optional**: DprE-1 ↔ DprE1, MDR-TB ↔ MDRTB
39
+ - **Period-optional**: M. tuberculosis ↔ M tuberculosis
40
+ - **Greek letters**: β-lactam ↔ beta-lactam
41
+
42
+ Word boundaries are enforced — "Rho" won't match inside "Rhodamine".
43
+
44
+ ### Phase 1b — Regex Patterns (Accession Numbers)
45
+ Seed entries in `accession_number.yml` auto-derive regex patterns for entire ID families:
46
+
47
+ | Seed | Auto-derived Pattern | Matches |
48
+ |---|---|---|
49
+ | `Rv0005` | `Rv\d{4}[c]?` | All Rv locus tags |
50
+ | `MT0005` | `MT\w+` | Mycobrowser IDs |
51
+ | `P9WGR1` | `[OPQ][0-9][A-Z0-9]{3}[0-9]` | UniProt accessions |
52
+ | `4TZK` | `[0-9][A-Z0-9]{3}` | PDB codes |
53
+ | `WP_003407354` | `WP_\d+` | NCBI RefSeq proteins |
54
+
55
+ ### Phase 2 — Fuzzy Match
56
+ Unmatched "entity-like" tokens (capitalized, contain digits, length ≥ 4) are compared against the dictionary using rapidfuzz. Catches typos and minor variants.
57
+
58
+ ```python
59
+ # Configurable threshold (0–100, default 85)
60
+ strict = FastNERExtractor(fuzzy_threshold=0) # disable fuzzy
61
+ lenient = FastNERExtractor(fuzzy_threshold=75) # more permissive
62
+ ```
63
+
64
+ ## Gazetteers
65
+
66
+ YAML files live in `structflo/ner/fast/gazetteers/`. Each file is a simple list of names — **nothing else**:
67
+
68
+ ```yaml
69
+ # target.yml
70
+ - InhA
71
+ - DprE1
72
+ - MmpL3
73
+ - AtpE
74
+ ```
75
+
76
+ The filename (without `.yml`) becomes the `entity_type`. Built-in gazetteers:
77
+
78
+ | File | Entity Type | Coverage |
79
+ |---|---|---|
80
+ | `target.yml` | target → `TargetEntity` | ~80 TB drug targets |
81
+ | `gene_name.yml` | gene_name → `TargetEntity` | ~75 Mtb gene names |
82
+ | `compound_name.yml` | compound_name → `ChemicalEntity` | ~50 TB compounds & abbreviations |
83
+ | `disease.yml` | disease → `DiseaseEntity` | TB disease variants |
84
+ | `accession_number.yml` | accession_number → `AccessionEntity` | Seed entries → regex patterns |
85
+ | `screening_method.yml` | screening_method → `ScreeningMethodEntity` | ~35 screening approaches |
86
+ | `functional_category.yml` | functional_category → `FunctionalCategoryEntity` | ~25 Mtb functional categories |
87
+ | `product.yml` | product → `ProductEntity` | ~35 gene product descriptions |
88
+
89
+ ## Adding New Gazetteers
90
+
91
+ ### Option 1: Add to existing files
92
+ Edit a YAML file and add names:
93
+
94
+ ```yaml
95
+ # target.yml
96
+ - InhA
97
+ - DprE1
98
+ - MyNewTarget # just add it
99
+ ```
100
+
101
+ ### Option 2: Create a new YAML file
102
+ Drop a new `.yml` file into any directory:
103
+
104
+ ```yaml
105
+ # my_gazetteers/assay.yml
106
+ - resazurin assay
107
+ - luciferase reporter assay
108
+ - disk diffusion assay
109
+ ```
110
+
111
+ ```python
112
+ fast = FastNERExtractor(gazetteer_dir="my_gazetteers/")
113
+ ```
114
+
115
+ ### Option 3: Add terms programmatically
116
+
117
+ ```python
118
+ fast = FastNERExtractor(
119
+ extra_gazetteers={
120
+ "target": ["NovelTarget1", "NovelTarget2"],
121
+ "compound_name": ["CompoundXYZ"],
122
+ }
123
+ )
124
+ ```
125
+
126
+ ## Output Compatibility
127
+
128
+ `FastNERExtractor` produces identical `NERResult` objects as the LLM-based `NERExtractor`. Everything downstream works the same:
129
+
130
+ ```python
131
+ result.all_entities() # flat list
132
+ result.to_dict() # serializable dict
133
+ result.to_dataframe() # pandas DataFrame
134
+ result.display() # interactive HTML
135
+ ```
136
+
137
+ Each entity includes `match_method` ("exact", "regex", or "fuzzy") and `canonical` (the gazetteer term it matched) in its `attributes` dict.
138
+
139
+ ## Fast vs LLM
140
+
141
+ | | `FastNERExtractor` | `NERExtractor` |
142
+ |---|---|---|
143
+ | Speed | ~1–5 ms per abstract | ~2–5 s per abstract |
144
+ | Novel entities | Only known terms | Discovers new entities |
145
+ | Context | String matching | Full contextual understanding |
146
+ | Cost | Free | API calls or GPU |
147
+ | Setup | Zero config | API key or Ollama |
148
+
149
+ **Recommended workflow**: Fast extractor as first pass (bulk screening), LLM extractor as second pass (deep analysis on interesting papers).
@@ -0,0 +1,22 @@
1
+ """Fast dictionary-based NER for TB drug discovery — no LLM required.
2
+
3
+ Quick start::
4
+
5
+ from structflo.ner.fast import FastNERExtractor
6
+
7
+ extractor = FastNERExtractor()
8
+ result = extractor.extract(
9
+ "Bedaquiline inhibits AtpE (Rv1305) in M. tuberculosis."
10
+ )
11
+ print(result.compounds)
12
+ print(result.targets)
13
+ df = result.to_dataframe()
14
+
15
+ Custom gazetteers::
16
+
17
+ extractor = FastNERExtractor(gazetteer_dir="/path/to/my/gazetteers")
18
+ """
19
+
20
+ from structflo.ner.fast.extractor import FastNERExtractor
21
+
22
+ __all__ = ["FastNERExtractor"]