rettxmutation 0.2.2__tar.gz → 0.3.3__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- rettxmutation-0.3.3/PKG-INFO +308 -0
- rettxmutation-0.3.3/README.md +282 -0
- {rettxmutation-0.2.2 → rettxmutation-0.3.3}/pyproject.toml +6 -2
- {rettxmutation-0.2.2 → rettxmutation-0.3.3}/src/rettxmutation/__init__.py +6 -1
- {rettxmutation-0.2.2 → rettxmutation-0.3.3}/src/rettxmutation/config.py +16 -0
- rettxmutation-0.3.3/src/rettxmutation/models/extraction_models.py +41 -0
- rettxmutation-0.3.3/src/rettxmutation/models/tool_outputs.py +91 -0
- rettxmutation-0.3.3/src/rettxmutation/prompts/__init__.py +0 -0
- rettxmutation-0.3.3/src/rettxmutation/prompts/extraction_agent.md +85 -0
- {rettxmutation-0.2.2 → rettxmutation-0.3.3}/src/rettxmutation/services/__init__.py +2 -0
- rettxmutation-0.3.3/src/rettxmutation/services/agent_service.py +556 -0
- rettxmutation-0.3.3/src/rettxmutation/services/agent_tools.py +257 -0
- {rettxmutation-0.2.2 → rettxmutation-0.3.3}/src/rettxmutation/services/services_factory.py +22 -0
- {rettxmutation-0.2.2 → rettxmutation-0.3.3}/src/rettxmutation/services/variant_validator_service.py +70 -2
- rettxmutation-0.3.3/src/rettxmutation/utils/phi_redactor.py +153 -0
- rettxmutation-0.3.3/src/rettxmutation.egg-info/PKG-INFO +308 -0
- {rettxmutation-0.2.2 → rettxmutation-0.3.3}/src/rettxmutation.egg-info/SOURCES.txt +11 -0
- rettxmutation-0.3.3/tests/test_agent_service.py +319 -0
- rettxmutation-0.3.3/tests/test_agent_tools.py +260 -0
- rettxmutation-0.3.3/tests/test_extraction_models.py +328 -0
- rettxmutation-0.3.3/tests/test_phi_redactor.py +137 -0
- rettxmutation-0.2.2/PKG-INFO +0 -212
- rettxmutation-0.2.2/README.md +0 -186
- rettxmutation-0.2.2/src/rettxmutation.egg-info/PKG-INFO +0 -212
- {rettxmutation-0.2.2 → rettxmutation-0.3.3}/LICENSE +0 -0
- {rettxmutation-0.2.2 → rettxmutation-0.3.3}/setup.cfg +0 -0
- {rettxmutation-0.2.2 → rettxmutation-0.3.3}/src/rettxmutation/models/__init__.py +0 -0
- {rettxmutation-0.2.2 → rettxmutation-0.3.3}/src/rettxmutation/models/gene_assembly.py +0 -0
- {rettxmutation-0.2.2 → rettxmutation-0.3.3}/src/rettxmutation/models/gene_models.py +0 -0
- {rettxmutation-0.2.2 → rettxmutation-0.3.3}/src/rettxmutation/models/gene_registry.py +0 -0
- {rettxmutation-0.2.2 → rettxmutation-0.3.3}/src/rettxmutation/models/keyword_collection.py +0 -0
- {rettxmutation-0.2.2 → rettxmutation-0.3.3}/src/rettxmutation/models/mutation_model.py +0 -0
- {rettxmutation-0.2.2 → rettxmutation-0.3.3}/src/rettxmutation/models/search_models.py +0 -0
- {rettxmutation-0.2.2 → rettxmutation-0.3.3}/src/rettxmutation/repositories/__init__.py +0 -0
- {rettxmutation-0.2.2 → rettxmutation-0.3.3}/src/rettxmutation/repositories/ai_search_repository.py +0 -0
- {rettxmutation-0.2.2 → rettxmutation-0.3.3}/src/rettxmutation/repositories/interfaces.py +0 -0
- {rettxmutation-0.2.2 → rettxmutation-0.3.3}/src/rettxmutation/repositories/text_analytics_repository.py +0 -0
- {rettxmutation-0.2.2 → rettxmutation-0.3.3}/src/rettxmutation/repositories/variant_validator_repository.py +0 -0
- {rettxmutation-0.2.2 → rettxmutation-0.3.3}/src/rettxmutation/services/ai_search.py +0 -0
- {rettxmutation-0.2.2 → rettxmutation-0.3.3}/src/rettxmutation/services/embedding_client.py +0 -0
- {rettxmutation-0.2.2 → rettxmutation-0.3.3}/src/rettxmutation/services/embedding_service.py +0 -0
- {rettxmutation-0.2.2 → rettxmutation-0.3.3}/src/rettxmutation/services/keyword_detector_service.py +0 -0
- {rettxmutation-0.2.2 → rettxmutation-0.3.3}/src/rettxmutation/services/mutation_tokenizator.py +0 -0
- {rettxmutation-0.2.2 → rettxmutation-0.3.3}/src/rettxmutation/services/variant_validator_mapping_service.py +0 -0
- {rettxmutation-0.2.2 → rettxmutation-0.3.3}/src/rettxmutation/utils/__init__.py +0 -0
- {rettxmutation-0.2.2 → rettxmutation-0.3.3}/src/rettxmutation/utils/hgvs_descriptor.py +0 -0
- {rettxmutation-0.2.2 → rettxmutation-0.3.3}/src/rettxmutation/utils/mutation_converter.py +0 -0
- {rettxmutation-0.2.2 → rettxmutation-0.3.3}/src/rettxmutation/utils/text_util.py +0 -0
- {rettxmutation-0.2.2 → rettxmutation-0.3.3}/src/rettxmutation.egg-info/dependency_links.txt +0 -0
- {rettxmutation-0.2.2 → rettxmutation-0.3.3}/src/rettxmutation.egg-info/requires.txt +0 -0
- {rettxmutation-0.2.2 → rettxmutation-0.3.3}/src/rettxmutation.egg-info/top_level.txt +0 -0
- {rettxmutation-0.2.2 → rettxmutation-0.3.3}/tests/test_gene_model.py +0 -0
- {rettxmutation-0.2.2 → rettxmutation-0.3.3}/tests/test_keyword_collection.py +0 -0
- {rettxmutation-0.2.2 → rettxmutation-0.3.3}/tests/test_keyword_detector.py +0 -0
- {rettxmutation-0.2.2 → rettxmutation-0.3.3}/tests/test_semantic_search.py +0 -0
- {rettxmutation-0.2.2 → rettxmutation-0.3.3}/tests/test_variant_validator.py +0 -0
|
@@ -0,0 +1,308 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: rettxmutation
|
|
3
|
+
Version: 0.3.3
|
|
4
|
+
Summary: Extract Rett Syndrome mutations from genetic diagnosis report
|
|
5
|
+
Author-email: Pedro Rocha <procha@rettsyndrome.eu>
|
|
6
|
+
License: MIT License
|
|
7
|
+
Project-URL: Homepage, https://github.com/rett-europe/rettxmutation
|
|
8
|
+
Project-URL: Issues, https://github.com/rett-europe/rettxmutation/issues
|
|
9
|
+
Classifier: Programming Language :: Python :: 3
|
|
10
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
11
|
+
Classifier: Operating System :: OS Independent
|
|
12
|
+
Requires-Python: >=3.8
|
|
13
|
+
Description-Content-Type: text/markdown
|
|
14
|
+
License-File: LICENSE
|
|
15
|
+
Requires-Dist: python-dotenv
|
|
16
|
+
Requires-Dist: azure-core
|
|
17
|
+
Requires-Dist: azure-ai-textanalytics
|
|
18
|
+
Requires-Dist: mutalyzer_hgvs_parser
|
|
19
|
+
Requires-Dist: pydantic
|
|
20
|
+
Requires-Dist: openai
|
|
21
|
+
Requires-Dist: backoff
|
|
22
|
+
Requires-Dist: jmespath
|
|
23
|
+
Requires-Dist: azure-search-documents
|
|
24
|
+
Requires-Dist: numpy
|
|
25
|
+
Dynamic: license-file
|
|
26
|
+
|
|
27
|
+
# RettX Mutation Analysis Library
|
|
28
|
+
|
|
29
|
+
[](https://badge.fury.io/py/rettxmutation)
|
|
30
|
+
[](https://www.python.org/downloads/)
|
|
31
|
+
[](https://opensource.org/licenses/MIT)
|
|
32
|
+
|
|
33
|
+
A Python library for extracting and validating genetic mutations from clinical reports using an AI-powered agentic pipeline. Supports **6 Rett Syndrome-related genes** across **multiple languages**, returning fully normalized HGVS nomenclature with genomic coordinates on both GRCh37 and GRCh38 assemblies.
|
|
34
|
+
|
|
35
|
+
## 🚀 Quick Start
|
|
36
|
+
|
|
37
|
+
### Installation
|
|
38
|
+
|
|
39
|
+
```bash
|
|
40
|
+
pip install rettxmutation
|
|
41
|
+
```
|
|
42
|
+
|
|
43
|
+
### Basic Usage
|
|
44
|
+
|
|
45
|
+
```python
|
|
46
|
+
import asyncio
|
|
47
|
+
from rettxmutation import RettxServices, DefaultConfig
|
|
48
|
+
|
|
49
|
+
async def extract():
|
|
50
|
+
config = DefaultConfig() # loads from .env / environment variables
|
|
51
|
+
|
|
52
|
+
with RettxServices(config) as services:
|
|
53
|
+
result = await services.agent_extraction_service.extract_mutations(
|
|
54
|
+
"The patient carries the mutation NM_004992.4:c.916C>T (p.Arg306Cys) in MECP2."
|
|
55
|
+
)
|
|
56
|
+
|
|
57
|
+
for key, mutation in result.mutations.items():
|
|
58
|
+
pt = mutation.primary_transcript
|
|
59
|
+
print(f"Gene: {pt.gene_id}")
|
|
60
|
+
print(f"Transcript: {pt.hgvs_transcript_variant}")
|
|
61
|
+
print(f"Protein: {pt.protein_consequence_tlr}")
|
|
62
|
+
print(f"Type: {mutation.variant_type}")
|
|
63
|
+
for assembly, coord in mutation.genomic_coordinates.items():
|
|
64
|
+
print(f"{assembly}: {coord.hgvs} (pos {coord.start:,}–{coord.end:,})")
|
|
65
|
+
|
|
66
|
+
asyncio.run(extract())
|
|
67
|
+
```
|
|
68
|
+
|
|
69
|
+
### CLI Example
|
|
70
|
+
|
|
71
|
+
```bash
|
|
72
|
+
python examples/extract_from_file.py path/to/genetic_report.txt --verbose
|
|
73
|
+
```
|
|
74
|
+
|
|
75
|
+
## ✨ Key Features
|
|
76
|
+
|
|
77
|
+
- **🧬 Multi-Gene Support**: MECP2, FOXG1, SLC6A1, CDKL5, EIF2B2, MEF2C — with curated RefSeq transcripts
|
|
78
|
+
- **🌍 Multilingual**: Processes reports in English, Spanish, Greek, Turkish, and more
|
|
79
|
+
- **🤖 Agentic Extraction**: Azure OpenAI-powered agent with tool-calling (gene registry lookup, variant validation, complex variant handling)
|
|
80
|
+
- **✅ HGVS Validation**: Every mutation is validated via VariantValidator with automatic coordinate liftover (GRCh37 ↔ GRCh38)
|
|
81
|
+
- **🔒 PHI Redaction**: Automatic removal of personal health information before LLM processing
|
|
82
|
+
- **⚡ Production Ready**: Type-safe Pydantic v2 models, exponential backoff, connection pooling
|
|
83
|
+
- **🔄 Dual Assembly Output**: Every mutation includes genomic coordinates on both GRCh37 and GRCh38
|
|
84
|
+
- **🏗️ Modular Architecture**: Lazy-initialized services with dependency injection and context manager support
|
|
85
|
+
|
|
86
|
+
## 🧬 Supported Genes
|
|
87
|
+
|
|
88
|
+
| Gene | Chromosome | Primary Transcript | Condition |
|
|
89
|
+
|------|-----------|-------------------|-----------|
|
|
90
|
+
| **MECP2** | Xq28 | NM_004992.4 (+NM_001110792.2) | Classic Rett Syndrome |
|
|
91
|
+
| **FOXG1** | 14q12 | NM_005249.5 | Congenital variant Rett |
|
|
92
|
+
| **SLC6A1** | 3p25.3 | NM_003042.4 | Myoclonic-atonic epilepsy |
|
|
93
|
+
| **CDKL5** | Xp22.13 | NM_001323289.2 | CDKL5 deficiency disorder |
|
|
94
|
+
| **EIF2B2** | 14q24.3 | NM_014239.4 | Vanishing white matter disease |
|
|
95
|
+
| **MEF2C** | 5q14.3 | NM_002397.5 (+NM_001131005.2) | MEF2C haploinsufficiency |
|
|
96
|
+
|
|
97
|
+
## 📊 Output Structure
|
|
98
|
+
|
|
99
|
+
The `ExtractionResult` contains:
|
|
100
|
+
|
|
101
|
+
```
|
|
102
|
+
ExtractionResult
|
|
103
|
+
├── mutations: Dict[str, GeneMutation] ← keyed by GRCh38 genomic HGVS
|
|
104
|
+
├── genes_detected: List[str] ← e.g. ["MECP2"]
|
|
105
|
+
├── extraction_log: List[str] ← agent reasoning trace
|
|
106
|
+
└── tool_calls_count: int ← total tool invocations
|
|
107
|
+
```
|
|
108
|
+
|
|
109
|
+
Each `GeneMutation` provides:
|
|
110
|
+
|
|
111
|
+
```
|
|
112
|
+
GeneMutation
|
|
113
|
+
├── genomic_coordinates:
|
|
114
|
+
│ ├── GRCh38: { assembly, hgvs, start, end, size }
|
|
115
|
+
│ └── GRCh37: { assembly, hgvs, start, end, size }
|
|
116
|
+
├── variant_type: "SNV" | "deletion" | "duplication" | "insertion" | "indel"
|
|
117
|
+
├── primary_transcript:
|
|
118
|
+
│ ├── gene_id, transcript_id
|
|
119
|
+
│ ├── hgvs_transcript_variant ← e.g. NM_004992.4:c.916C>T
|
|
120
|
+
│ ├── protein_consequence_tlr ← e.g. NP_004983.1:p.(Arg306Cys)
|
|
121
|
+
│ └── protein_consequence_slr ← e.g. NP_004983.1:p.(R306C)
|
|
122
|
+
└── secondary_transcript (optional)
|
|
123
|
+
```
|
|
124
|
+
|
|
125
|
+
## 🛠️ Requirements
|
|
126
|
+
|
|
127
|
+
### Python Version
|
|
128
|
+
- Python 3.8 or higher
|
|
129
|
+
|
|
130
|
+
### Azure Services
|
|
131
|
+
|
|
132
|
+
| Service | Required? | Purpose |
|
|
133
|
+
|---------|-----------|---------|
|
|
134
|
+
| **Azure OpenAI** | ✅ Required | Agentic mutation extraction |
|
|
135
|
+
| **Azure AI Search** | Optional | Semantic search for keyword detection |
|
|
136
|
+
| **Azure Cognitive Services** | Optional | Text analytics enrichment |
|
|
137
|
+
|
|
138
|
+
### Environment Variables
|
|
139
|
+
|
|
140
|
+
```bash
|
|
141
|
+
# Required — Azure OpenAI
|
|
142
|
+
RETTX_OPENAI_ENDPOINT=https://your-openai.openai.azure.com/
|
|
143
|
+
RETTX_OPENAI_KEY=your-openai-key
|
|
144
|
+
RETTX_OPENAI_MODEL_NAME=gpt-4o # deployment name
|
|
145
|
+
|
|
146
|
+
# Optional — Agent model (defaults to RETTX_OPENAI_MODEL_NAME if not set)
|
|
147
|
+
RETTX_OPENAI_AGENT_DEPLOYMENT=gpt-4o # agent-specific deployment
|
|
148
|
+
RETTX_OPENAI_AGENT_MODEL_VERSION=2024-11-20
|
|
149
|
+
|
|
150
|
+
# Optional — Embeddings
|
|
151
|
+
RETTX_EMBEDDING_DEPLOYMENT=text-embedding-ada-002
|
|
152
|
+
|
|
153
|
+
# Optional — Azure AI Search
|
|
154
|
+
RETTX_AI_SEARCH_SERVICE=your-search-service
|
|
155
|
+
RETTX_AI_SEARCH_API_KEY=your-search-key
|
|
156
|
+
RETTX_AI_SEARCH_INDEX_NAME=your-index-name
|
|
157
|
+
|
|
158
|
+
# Optional — Azure Cognitive Services
|
|
159
|
+
RETTX_COGNITIVE_SERVICES_ENDPOINT=https://your-cognitive-services.cognitiveservices.azure.com/
|
|
160
|
+
RETTX_COGNITIVE_SERVICES_KEY=your-cognitive-services-key
|
|
161
|
+
```
|
|
162
|
+
|
|
163
|
+
## 📋 Processing Pipeline
|
|
164
|
+
|
|
165
|
+
The agentic extraction pipeline works as follows:
|
|
166
|
+
|
|
167
|
+
```
|
|
168
|
+
Input Text (any language)
|
|
169
|
+
│
|
|
170
|
+
▼
|
|
171
|
+
┌──────────────────────┐
|
|
172
|
+
│ PHI Redaction │ Remove patient names, DOBs, IDs
|
|
173
|
+
└──────────┬───────────┘
|
|
174
|
+
▼
|
|
175
|
+
┌──────────────────────┐
|
|
176
|
+
│ AI Agent (OpenAI) │ Reads text, identifies mutations
|
|
177
|
+
│ │
|
|
178
|
+
│ Tools available: │
|
|
179
|
+
│ • lookup_gene_registry → gene info + RefSeq transcripts
|
|
180
|
+
│ • validate_variant → HGVS validation + coordinates
|
|
181
|
+
│ • validate_complex → CNV / genomic coordinate validation
|
|
182
|
+
└──────────┬───────────┘
|
|
183
|
+
▼
|
|
184
|
+
┌──────────────────────┐
|
|
185
|
+
│ Structured Output │ ExtractionResult with validated
|
|
186
|
+
│ │ GeneMutation objects
|
|
187
|
+
└──────────────────────┘
|
|
188
|
+
```
|
|
189
|
+
|
|
190
|
+
Key capabilities:
|
|
191
|
+
- **Ensembl → RefSeq remapping**: Handles reports using Ensembl transcripts (ENST*) by using genomic coordinates
|
|
192
|
+
- **Minus-strand awareness**: Correctly complements alleles for genes on the reverse strand
|
|
193
|
+
- **Old nomenclature**: Normalizes legacy formats (e.g., `502C->T`, `R168X`) to current HGVS
|
|
194
|
+
|
|
195
|
+
## 💻 Available Services
|
|
196
|
+
|
|
197
|
+
All services are lazily initialized via `RettxServices`:
|
|
198
|
+
|
|
199
|
+
```python
|
|
200
|
+
with RettxServices(config) as services:
|
|
201
|
+
# Core extraction
|
|
202
|
+
services.agent_extraction_service # AI-powered mutation extraction
|
|
203
|
+
|
|
204
|
+
# Validation & analysis
|
|
205
|
+
services.variant_validator_service # HGVS validation + coordinate liftover
|
|
206
|
+
services.mutation_tokenizator # Mutation string tokenization
|
|
207
|
+
|
|
208
|
+
# Search & embeddings
|
|
209
|
+
services.embedding_service # Azure OpenAI embeddings
|
|
210
|
+
services.ai_search_service # Azure AI Search integration
|
|
211
|
+
services.keyword_detector_service # Multi-layer keyword detection
|
|
212
|
+
```
|
|
213
|
+
|
|
214
|
+
### Direct Variant Validation
|
|
215
|
+
|
|
216
|
+
```python
|
|
217
|
+
with RettxServices(config) as services:
|
|
218
|
+
vvs = services.variant_validator_service
|
|
219
|
+
|
|
220
|
+
# Validate a transcript-level variant
|
|
221
|
+
result = vvs.get_gene_mutation_from_transcript("NM_004992.4:c.916C>T")
|
|
222
|
+
print(result.primary_transcript.protein_consequence_tlr)
|
|
223
|
+
# → NP_004983.1:p.(Arg306Cys)
|
|
224
|
+
|
|
225
|
+
# Validate a complex / genomic variant
|
|
226
|
+
result = vvs.create_gene_mutation_from_complex_variant(
|
|
227
|
+
assembly_build="GRCh38",
|
|
228
|
+
assembly_refseq="NC_000023.11",
|
|
229
|
+
variant_description="NC_000023.11:g.154030912G>A",
|
|
230
|
+
gene_symbol="MECP2"
|
|
231
|
+
)
|
|
232
|
+
```
|
|
233
|
+
|
|
234
|
+
### Custom Configuration
|
|
235
|
+
|
|
236
|
+
```python
|
|
237
|
+
class MyConfig:
|
|
238
|
+
"""Custom configuration for production (e.g., from Key Vault)."""
|
|
239
|
+
RETTX_OPENAI_ENDPOINT = "https://my-openai.openai.azure.com/"
|
|
240
|
+
RETTX_OPENAI_KEY = get_secret("openai-key")
|
|
241
|
+
RETTX_OPENAI_MODEL_NAME = "gpt-4o"
|
|
242
|
+
# Only set fields needed for the services you use
|
|
243
|
+
|
|
244
|
+
with RettxServices(MyConfig()) as services:
|
|
245
|
+
result = await services.agent_extraction_service.extract_mutations(text)
|
|
246
|
+
```
|
|
247
|
+
|
|
248
|
+
## 🧪 Golden Test Suite
|
|
249
|
+
|
|
250
|
+
The library includes a comprehensive golden test suite with 11 real-world genetic reports:
|
|
251
|
+
|
|
252
|
+
| Gene | Variant Type | Language | Key Feature |
|
|
253
|
+
|------|-------------|----------|-------------|
|
|
254
|
+
| MECP2 | SNV, splicing, deletion, duplication | EN, ES, EL, TR | Multiple transcripts |
|
|
255
|
+
| FOXG1 | Frameshift deletion | TR | Non-MECP2 gene |
|
|
256
|
+
| SLC6A1 | Whole-gene CNV (~20kb) | ES | Copy number variant |
|
|
257
|
+
|
|
258
|
+
Run golden tests:
|
|
259
|
+
```bash
|
|
260
|
+
# Mock mode (no API calls, uses recorded responses)
|
|
261
|
+
python -m pytest tests/golden/ --golden-mode=mock -v
|
|
262
|
+
|
|
263
|
+
# Live mode (calls real APIs)
|
|
264
|
+
python -m pytest tests/golden/ --golden-mode=live -v
|
|
265
|
+
```
|
|
266
|
+
|
|
267
|
+
## 🎯 Use Cases
|
|
268
|
+
|
|
269
|
+
- **🏥 Clinical Genetics**: Extract mutations from diagnostic reports in any language
|
|
270
|
+
- **🔬 Research**: Analyze genetic data across Rett Syndrome and related conditions
|
|
271
|
+
- **📊 Patient Registries**: Populate genetic databases with normalized HGVS nomenclature
|
|
272
|
+
- **🤖 Bioinformatics Pipelines**: Integrate as a library or via the CLI example
|
|
273
|
+
- **📱 Clinical Applications**: Build tools with structured mutation data (dual-assembly coordinates)
|
|
274
|
+
|
|
275
|
+
## 🔧 Reliability
|
|
276
|
+
|
|
277
|
+
- **Exponential Backoff**: Automatic retry for VariantValidator and OpenAI API calls
|
|
278
|
+
- **Graceful Degradation**: Optional services (AI Search, Cognitive Services) degrade gracefully
|
|
279
|
+
- **PHI Redaction**: Patient data is stripped before any LLM processing
|
|
280
|
+
- **Type Safety**: Pydantic v2 models with runtime validation
|
|
281
|
+
- **Context Manager**: Automatic resource cleanup via `with` statement
|
|
282
|
+
- **Comprehensive Logging**: Structured extraction logs with tool call traces
|
|
283
|
+
|
|
284
|
+
## 🤝 Contributing
|
|
285
|
+
|
|
286
|
+
We welcome contributions! Please see our [GitHub repository](https://github.com/rett-europe/rettxmutation) for:
|
|
287
|
+
- Issue reporting
|
|
288
|
+
- Feature requests
|
|
289
|
+
- Pull request guidelines
|
|
290
|
+
- Development setup instructions
|
|
291
|
+
|
|
292
|
+
## 📄 License
|
|
293
|
+
|
|
294
|
+
This project is licensed under the MIT License - see the [LICENSE](LICENSE) file for details.
|
|
295
|
+
|
|
296
|
+
## 🆘 Support
|
|
297
|
+
|
|
298
|
+
- **Issues**: [GitHub Issues](https://github.com/rett-europe/rettxmutation/issues)
|
|
299
|
+
- **Documentation**: [API Documentation](https://github.com/rett-europe/rettxmutation)
|
|
300
|
+
- **Contact**: procha@rettsyndrome.eu
|
|
301
|
+
|
|
302
|
+
## 🔮 Roadmap
|
|
303
|
+
|
|
304
|
+
- **Additional Genes**: Expand the gene registry beyond the current 6 genes
|
|
305
|
+
- **Batch Processing**: Process multiple reports in parallel with rate limiting
|
|
306
|
+
- **Confidence Scoring**: Per-mutation confidence metrics based on report quality
|
|
307
|
+
- **Structured Report Parsing**: Native support for VCF, JSON, and HL7 FHIR formats
|
|
308
|
+
- **Cloud Deployment**: Docker containers and Azure deployment templates
|
|
@@ -0,0 +1,282 @@
|
|
|
1
|
+
# RettX Mutation Analysis Library
|
|
2
|
+
|
|
3
|
+
[](https://badge.fury.io/py/rettxmutation)
|
|
4
|
+
[](https://www.python.org/downloads/)
|
|
5
|
+
[](https://opensource.org/licenses/MIT)
|
|
6
|
+
|
|
7
|
+
A Python library for extracting and validating genetic mutations from clinical reports using an AI-powered agentic pipeline. Supports **6 Rett Syndrome-related genes** across **multiple languages**, returning fully normalized HGVS nomenclature with genomic coordinates on both GRCh37 and GRCh38 assemblies.
|
|
8
|
+
|
|
9
|
+
## 🚀 Quick Start
|
|
10
|
+
|
|
11
|
+
### Installation
|
|
12
|
+
|
|
13
|
+
```bash
|
|
14
|
+
pip install rettxmutation
|
|
15
|
+
```
|
|
16
|
+
|
|
17
|
+
### Basic Usage
|
|
18
|
+
|
|
19
|
+
```python
|
|
20
|
+
import asyncio
|
|
21
|
+
from rettxmutation import RettxServices, DefaultConfig
|
|
22
|
+
|
|
23
|
+
async def extract():
|
|
24
|
+
config = DefaultConfig() # loads from .env / environment variables
|
|
25
|
+
|
|
26
|
+
with RettxServices(config) as services:
|
|
27
|
+
result = await services.agent_extraction_service.extract_mutations(
|
|
28
|
+
"The patient carries the mutation NM_004992.4:c.916C>T (p.Arg306Cys) in MECP2."
|
|
29
|
+
)
|
|
30
|
+
|
|
31
|
+
for key, mutation in result.mutations.items():
|
|
32
|
+
pt = mutation.primary_transcript
|
|
33
|
+
print(f"Gene: {pt.gene_id}")
|
|
34
|
+
print(f"Transcript: {pt.hgvs_transcript_variant}")
|
|
35
|
+
print(f"Protein: {pt.protein_consequence_tlr}")
|
|
36
|
+
print(f"Type: {mutation.variant_type}")
|
|
37
|
+
for assembly, coord in mutation.genomic_coordinates.items():
|
|
38
|
+
print(f"{assembly}: {coord.hgvs} (pos {coord.start:,}–{coord.end:,})")
|
|
39
|
+
|
|
40
|
+
asyncio.run(extract())
|
|
41
|
+
```
|
|
42
|
+
|
|
43
|
+
### CLI Example
|
|
44
|
+
|
|
45
|
+
```bash
|
|
46
|
+
python examples/extract_from_file.py path/to/genetic_report.txt --verbose
|
|
47
|
+
```
|
|
48
|
+
|
|
49
|
+
## ✨ Key Features
|
|
50
|
+
|
|
51
|
+
- **🧬 Multi-Gene Support**: MECP2, FOXG1, SLC6A1, CDKL5, EIF2B2, MEF2C — with curated RefSeq transcripts
|
|
52
|
+
- **🌍 Multilingual**: Processes reports in English, Spanish, Greek, Turkish, and more
|
|
53
|
+
- **🤖 Agentic Extraction**: Azure OpenAI-powered agent with tool-calling (gene registry lookup, variant validation, complex variant handling)
|
|
54
|
+
- **✅ HGVS Validation**: Every mutation is validated via VariantValidator with automatic coordinate liftover (GRCh37 ↔ GRCh38)
|
|
55
|
+
- **🔒 PHI Redaction**: Automatic removal of personal health information before LLM processing
|
|
56
|
+
- **⚡ Production Ready**: Type-safe Pydantic v2 models, exponential backoff, connection pooling
|
|
57
|
+
- **🔄 Dual Assembly Output**: Every mutation includes genomic coordinates on both GRCh37 and GRCh38
|
|
58
|
+
- **🏗️ Modular Architecture**: Lazy-initialized services with dependency injection and context manager support
|
|
59
|
+
|
|
60
|
+
## 🧬 Supported Genes
|
|
61
|
+
|
|
62
|
+
| Gene | Chromosome | Primary Transcript | Condition |
|
|
63
|
+
|------|-----------|-------------------|-----------|
|
|
64
|
+
| **MECP2** | Xq28 | NM_004992.4 (+NM_001110792.2) | Classic Rett Syndrome |
|
|
65
|
+
| **FOXG1** | 14q12 | NM_005249.5 | Congenital variant Rett |
|
|
66
|
+
| **SLC6A1** | 3p25.3 | NM_003042.4 | Myoclonic-atonic epilepsy |
|
|
67
|
+
| **CDKL5** | Xp22.13 | NM_001323289.2 | CDKL5 deficiency disorder |
|
|
68
|
+
| **EIF2B2** | 14q24.3 | NM_014239.4 | Vanishing white matter disease |
|
|
69
|
+
| **MEF2C** | 5q14.3 | NM_002397.5 (+NM_001131005.2) | MEF2C haploinsufficiency |
|
|
70
|
+
|
|
71
|
+
## 📊 Output Structure
|
|
72
|
+
|
|
73
|
+
The `ExtractionResult` contains:
|
|
74
|
+
|
|
75
|
+
```
|
|
76
|
+
ExtractionResult
|
|
77
|
+
├── mutations: Dict[str, GeneMutation] ← keyed by GRCh38 genomic HGVS
|
|
78
|
+
├── genes_detected: List[str] ← e.g. ["MECP2"]
|
|
79
|
+
├── extraction_log: List[str] ← agent reasoning trace
|
|
80
|
+
└── tool_calls_count: int ← total tool invocations
|
|
81
|
+
```
|
|
82
|
+
|
|
83
|
+
Each `GeneMutation` provides:
|
|
84
|
+
|
|
85
|
+
```
|
|
86
|
+
GeneMutation
|
|
87
|
+
├── genomic_coordinates:
|
|
88
|
+
│ ├── GRCh38: { assembly, hgvs, start, end, size }
|
|
89
|
+
│ └── GRCh37: { assembly, hgvs, start, end, size }
|
|
90
|
+
├── variant_type: "SNV" | "deletion" | "duplication" | "insertion" | "indel"
|
|
91
|
+
├── primary_transcript:
|
|
92
|
+
│ ├── gene_id, transcript_id
|
|
93
|
+
│ ├── hgvs_transcript_variant ← e.g. NM_004992.4:c.916C>T
|
|
94
|
+
│ ├── protein_consequence_tlr ← e.g. NP_004983.1:p.(Arg306Cys)
|
|
95
|
+
│ └── protein_consequence_slr ← e.g. NP_004983.1:p.(R306C)
|
|
96
|
+
└── secondary_transcript (optional)
|
|
97
|
+
```
|
|
98
|
+
|
|
99
|
+
## 🛠️ Requirements
|
|
100
|
+
|
|
101
|
+
### Python Version
|
|
102
|
+
- Python 3.8 or higher
|
|
103
|
+
|
|
104
|
+
### Azure Services
|
|
105
|
+
|
|
106
|
+
| Service | Required? | Purpose |
|
|
107
|
+
|---------|-----------|---------|
|
|
108
|
+
| **Azure OpenAI** | ✅ Required | Agentic mutation extraction |
|
|
109
|
+
| **Azure AI Search** | Optional | Semantic search for keyword detection |
|
|
110
|
+
| **Azure Cognitive Services** | Optional | Text analytics enrichment |
|
|
111
|
+
|
|
112
|
+
### Environment Variables
|
|
113
|
+
|
|
114
|
+
```bash
|
|
115
|
+
# Required — Azure OpenAI
|
|
116
|
+
RETTX_OPENAI_ENDPOINT=https://your-openai.openai.azure.com/
|
|
117
|
+
RETTX_OPENAI_KEY=your-openai-key
|
|
118
|
+
RETTX_OPENAI_MODEL_NAME=gpt-4o # deployment name
|
|
119
|
+
|
|
120
|
+
# Optional — Agent model (defaults to RETTX_OPENAI_MODEL_NAME if not set)
|
|
121
|
+
RETTX_OPENAI_AGENT_DEPLOYMENT=gpt-4o # agent-specific deployment
|
|
122
|
+
RETTX_OPENAI_AGENT_MODEL_VERSION=2024-11-20
|
|
123
|
+
|
|
124
|
+
# Optional — Embeddings
|
|
125
|
+
RETTX_EMBEDDING_DEPLOYMENT=text-embedding-ada-002
|
|
126
|
+
|
|
127
|
+
# Optional — Azure AI Search
|
|
128
|
+
RETTX_AI_SEARCH_SERVICE=your-search-service
|
|
129
|
+
RETTX_AI_SEARCH_API_KEY=your-search-key
|
|
130
|
+
RETTX_AI_SEARCH_INDEX_NAME=your-index-name
|
|
131
|
+
|
|
132
|
+
# Optional — Azure Cognitive Services
|
|
133
|
+
RETTX_COGNITIVE_SERVICES_ENDPOINT=https://your-cognitive-services.cognitiveservices.azure.com/
|
|
134
|
+
RETTX_COGNITIVE_SERVICES_KEY=your-cognitive-services-key
|
|
135
|
+
```
|
|
136
|
+
|
|
137
|
+
## 📋 Processing Pipeline
|
|
138
|
+
|
|
139
|
+
The agentic extraction pipeline works as follows:
|
|
140
|
+
|
|
141
|
+
```
|
|
142
|
+
Input Text (any language)
|
|
143
|
+
│
|
|
144
|
+
▼
|
|
145
|
+
┌──────────────────────┐
|
|
146
|
+
│ PHI Redaction │ Remove patient names, DOBs, IDs
|
|
147
|
+
└──────────┬───────────┘
|
|
148
|
+
▼
|
|
149
|
+
┌──────────────────────┐
|
|
150
|
+
│ AI Agent (OpenAI) │ Reads text, identifies mutations
|
|
151
|
+
│ │
|
|
152
|
+
│ Tools available: │
|
|
153
|
+
│ • lookup_gene_registry → gene info + RefSeq transcripts
|
|
154
|
+
│ • validate_variant → HGVS validation + coordinates
|
|
155
|
+
│ • validate_complex → CNV / genomic coordinate validation
|
|
156
|
+
└──────────┬───────────┘
|
|
157
|
+
▼
|
|
158
|
+
┌──────────────────────┐
|
|
159
|
+
│ Structured Output │ ExtractionResult with validated
|
|
160
|
+
│ │ GeneMutation objects
|
|
161
|
+
└──────────────────────┘
|
|
162
|
+
```
|
|
163
|
+
|
|
164
|
+
Key capabilities:
|
|
165
|
+
- **Ensembl → RefSeq remapping**: Handles reports using Ensembl transcripts (ENST*) by using genomic coordinates
|
|
166
|
+
- **Minus-strand awareness**: Correctly complements alleles for genes on the reverse strand
|
|
167
|
+
- **Old nomenclature**: Normalizes legacy formats (e.g., `502C->T`, `R168X`) to current HGVS
|
|
168
|
+
|
|
169
|
+
## 💻 Available Services
|
|
170
|
+
|
|
171
|
+
All services are lazily initialized via `RettxServices`:
|
|
172
|
+
|
|
173
|
+
```python
|
|
174
|
+
with RettxServices(config) as services:
|
|
175
|
+
# Core extraction
|
|
176
|
+
services.agent_extraction_service # AI-powered mutation extraction
|
|
177
|
+
|
|
178
|
+
# Validation & analysis
|
|
179
|
+
services.variant_validator_service # HGVS validation + coordinate liftover
|
|
180
|
+
services.mutation_tokenizator # Mutation string tokenization
|
|
181
|
+
|
|
182
|
+
# Search & embeddings
|
|
183
|
+
services.embedding_service # Azure OpenAI embeddings
|
|
184
|
+
services.ai_search_service # Azure AI Search integration
|
|
185
|
+
services.keyword_detector_service # Multi-layer keyword detection
|
|
186
|
+
```
|
|
187
|
+
|
|
188
|
+
### Direct Variant Validation
|
|
189
|
+
|
|
190
|
+
```python
|
|
191
|
+
with RettxServices(config) as services:
|
|
192
|
+
vvs = services.variant_validator_service
|
|
193
|
+
|
|
194
|
+
# Validate a transcript-level variant
|
|
195
|
+
result = vvs.get_gene_mutation_from_transcript("NM_004992.4:c.916C>T")
|
|
196
|
+
print(result.primary_transcript.protein_consequence_tlr)
|
|
197
|
+
# → NP_004983.1:p.(Arg306Cys)
|
|
198
|
+
|
|
199
|
+
# Validate a complex / genomic variant
|
|
200
|
+
result = vvs.create_gene_mutation_from_complex_variant(
|
|
201
|
+
assembly_build="GRCh38",
|
|
202
|
+
assembly_refseq="NC_000023.11",
|
|
203
|
+
variant_description="NC_000023.11:g.154030912G>A",
|
|
204
|
+
gene_symbol="MECP2"
|
|
205
|
+
)
|
|
206
|
+
```
|
|
207
|
+
|
|
208
|
+
### Custom Configuration
|
|
209
|
+
|
|
210
|
+
```python
|
|
211
|
+
class MyConfig:
|
|
212
|
+
"""Custom configuration for production (e.g., from Key Vault)."""
|
|
213
|
+
RETTX_OPENAI_ENDPOINT = "https://my-openai.openai.azure.com/"
|
|
214
|
+
RETTX_OPENAI_KEY = get_secret("openai-key")
|
|
215
|
+
RETTX_OPENAI_MODEL_NAME = "gpt-4o"
|
|
216
|
+
# Only set fields needed for the services you use
|
|
217
|
+
|
|
218
|
+
with RettxServices(MyConfig()) as services:
|
|
219
|
+
result = await services.agent_extraction_service.extract_mutations(text)
|
|
220
|
+
```
|
|
221
|
+
|
|
222
|
+
## 🧪 Golden Test Suite
|
|
223
|
+
|
|
224
|
+
The library includes a comprehensive golden test suite with 11 real-world genetic reports:
|
|
225
|
+
|
|
226
|
+
| Gene | Variant Type | Language | Key Feature |
|
|
227
|
+
|------|-------------|----------|-------------|
|
|
228
|
+
| MECP2 | SNV, splicing, deletion, duplication | EN, ES, EL, TR | Multiple transcripts |
|
|
229
|
+
| FOXG1 | Frameshift deletion | TR | Non-MECP2 gene |
|
|
230
|
+
| SLC6A1 | Whole-gene CNV (~20kb) | ES | Copy number variant |
|
|
231
|
+
|
|
232
|
+
Run golden tests:
|
|
233
|
+
```bash
|
|
234
|
+
# Mock mode (no API calls, uses recorded responses)
|
|
235
|
+
python -m pytest tests/golden/ --golden-mode=mock -v
|
|
236
|
+
|
|
237
|
+
# Live mode (calls real APIs)
|
|
238
|
+
python -m pytest tests/golden/ --golden-mode=live -v
|
|
239
|
+
```
|
|
240
|
+
|
|
241
|
+
## 🎯 Use Cases
|
|
242
|
+
|
|
243
|
+
- **🏥 Clinical Genetics**: Extract mutations from diagnostic reports in any language
|
|
244
|
+
- **🔬 Research**: Analyze genetic data across Rett Syndrome and related conditions
|
|
245
|
+
- **📊 Patient Registries**: Populate genetic databases with normalized HGVS nomenclature
|
|
246
|
+
- **🤖 Bioinformatics Pipelines**: Integrate as a library or via the CLI example
|
|
247
|
+
- **📱 Clinical Applications**: Build tools with structured mutation data (dual-assembly coordinates)
|
|
248
|
+
|
|
249
|
+
## 🔧 Reliability
|
|
250
|
+
|
|
251
|
+
- **Exponential Backoff**: Automatic retry for VariantValidator and OpenAI API calls
|
|
252
|
+
- **Graceful Degradation**: Optional services (AI Search, Cognitive Services) degrade gracefully
|
|
253
|
+
- **PHI Redaction**: Patient data is stripped before any LLM processing
|
|
254
|
+
- **Type Safety**: Pydantic v2 models with runtime validation
|
|
255
|
+
- **Context Manager**: Automatic resource cleanup via `with` statement
|
|
256
|
+
- **Comprehensive Logging**: Structured extraction logs with tool call traces
|
|
257
|
+
|
|
258
|
+
## 🤝 Contributing
|
|
259
|
+
|
|
260
|
+
We welcome contributions! Please see our [GitHub repository](https://github.com/rett-europe/rettxmutation) for:
|
|
261
|
+
- Issue reporting
|
|
262
|
+
- Feature requests
|
|
263
|
+
- Pull request guidelines
|
|
264
|
+
- Development setup instructions
|
|
265
|
+
|
|
266
|
+
## 📄 License
|
|
267
|
+
|
|
268
|
+
This project is licensed under the MIT License - see the [LICENSE](LICENSE) file for details.
|
|
269
|
+
|
|
270
|
+
## 🆘 Support
|
|
271
|
+
|
|
272
|
+
- **Issues**: [GitHub Issues](https://github.com/rett-europe/rettxmutation/issues)
|
|
273
|
+
- **Documentation**: [API Documentation](https://github.com/rett-europe/rettxmutation)
|
|
274
|
+
- **Contact**: procha@rettsyndrome.eu
|
|
275
|
+
|
|
276
|
+
## 🔮 Roadmap
|
|
277
|
+
|
|
278
|
+
- **Additional Genes**: Expand the gene registry beyond the current 6 genes
|
|
279
|
+
- **Batch Processing**: Process multiple reports in parallel with rate limiting
|
|
280
|
+
- **Confidence Scoring**: Per-mutation confidence metrics based on report quality
|
|
281
|
+
- **Structured Report Parsing**: Native support for VCF, JSON, and HL7 FHIR formats
|
|
282
|
+
- **Cloud Deployment**: Docker containers and Azure deployment templates
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "rettxmutation"
|
|
3
|
-
version = "0.
|
|
3
|
+
version = "0.3.3"
|
|
4
4
|
authors = [
|
|
5
5
|
{ name="Pedro Rocha", email="procha@rettsyndrome.eu" },
|
|
6
6
|
]
|
|
@@ -36,7 +36,11 @@ package-dir = {"" = "src"}
|
|
|
36
36
|
packages = [
|
|
37
37
|
"rettxmutation",
|
|
38
38
|
"rettxmutation.models",
|
|
39
|
+
"rettxmutation.prompts",
|
|
39
40
|
"rettxmutation.repositories",
|
|
40
41
|
"rettxmutation.services",
|
|
41
42
|
"rettxmutation.utils"
|
|
42
|
-
]
|
|
43
|
+
]
|
|
44
|
+
|
|
45
|
+
[tool.setuptools.package-data]
|
|
46
|
+
rettxmutation = ["prompts/*.md"]
|
|
@@ -29,9 +29,11 @@ from .services import (
|
|
|
29
29
|
VariantValidatorService,
|
|
30
30
|
MutationTokenizator,
|
|
31
31
|
AISearchService,
|
|
32
|
+
AgentExtractionService,
|
|
32
33
|
)
|
|
34
|
+
from .models.extraction_models import ExtractionResult
|
|
33
35
|
|
|
34
|
-
__version__ = "0.
|
|
36
|
+
__version__ = "0.3.3"
|
|
35
37
|
|
|
36
38
|
__all__ = [
|
|
37
39
|
# Core configuration
|
|
@@ -46,6 +48,9 @@ __all__ = [
|
|
|
46
48
|
"VariantValidatorService",
|
|
47
49
|
"MutationTokenizator",
|
|
48
50
|
"AISearchService",
|
|
51
|
+
"AgentExtractionService",
|
|
52
|
+
# Models
|
|
53
|
+
"ExtractionResult",
|
|
49
54
|
|
|
50
55
|
# Version
|
|
51
56
|
"__version__"
|
|
@@ -34,6 +34,10 @@ class RettxConfig(Protocol):
|
|
|
34
34
|
RETTX_OPENAI_MODEL_NAME: Optional[str] = None
|
|
35
35
|
RETTX_EMBEDDING_DEPLOYMENT: Optional[str] = None
|
|
36
36
|
|
|
37
|
+
# Azure OpenAI Agent Configuration (for agentic extraction)
|
|
38
|
+
RETTX_OPENAI_AGENT_DEPLOYMENT: Optional[str] = None
|
|
39
|
+
RETTX_OPENAI_AGENT_MODEL_VERSION: Optional[str] = None
|
|
40
|
+
|
|
37
41
|
# Azure Cognitive Services (Text Analytics)
|
|
38
42
|
RETTX_COGNITIVE_SERVICES_ENDPOINT: Optional[str] = None
|
|
39
43
|
RETTX_COGNITIVE_SERVICES_KEY: Optional[str] = None
|
|
@@ -106,6 +110,18 @@ class DefaultConfig:
|
|
|
106
110
|
self.RETTX_OPENAI_MODEL_NAME = os.getenv('RETTX_OPENAI_MODEL_NAME')
|
|
107
111
|
self.RETTX_EMBEDDING_DEPLOYMENT = os.getenv('RETTX_EMBEDDING_DEPLOYMENT')
|
|
108
112
|
|
|
113
|
+
# Azure OpenAI Agent Configuration (for agentic extraction)
|
|
114
|
+
# Falls back to RETTX_OPENAI_MODEL_NAME if AGENT_DEPLOYMENT not set
|
|
115
|
+
self.RETTX_OPENAI_AGENT_DEPLOYMENT = (
|
|
116
|
+
os.getenv('RETTX_OPENAI_AGENT_DEPLOYMENT')
|
|
117
|
+
or os.getenv('RETTX_OPENAI_MODEL_NAME')
|
|
118
|
+
)
|
|
119
|
+
self.RETTX_OPENAI_AGENT_MODEL_VERSION = (
|
|
120
|
+
os.getenv('RETTX_OPENAI_AGENT_MODEL_VERSION')
|
|
121
|
+
or os.getenv('RETTX_OPENAI_MODEL_VERSION')
|
|
122
|
+
or '2024-11-20'
|
|
123
|
+
)
|
|
124
|
+
|
|
109
125
|
# Azure Cognitive Services
|
|
110
126
|
self.RETTX_COGNITIVE_SERVICES_ENDPOINT = os.getenv('RETTX_COGNITIVE_SERVICES_ENDPOINT')
|
|
111
127
|
self.RETTX_COGNITIVE_SERVICES_KEY = os.getenv('RETTX_COGNITIVE_SERVICES_KEY')
|