vernacbridge 1.0.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- vernacbridge-1.0.0/LICENSE +21 -0
- vernacbridge-1.0.0/PKG-INFO +217 -0
- vernacbridge-1.0.0/README.md +187 -0
- vernacbridge-1.0.0/pyproject.toml +67 -0
- vernacbridge-1.0.0/setup.cfg +4 -0
- vernacbridge-1.0.0/tests/test_cleaner.py +105 -0
- vernacbridge-1.0.0/tests/test_context.py +256 -0
- vernacbridge-1.0.0/tests/test_core.py +73 -0
- vernacbridge-1.0.0/tests/test_dataset_engine.py +157 -0
- vernacbridge-1.0.0/tests/test_integration_real_dataset.py +131 -0
- vernacbridge-1.0.0/tests/test_intelligence.py +145 -0
- vernacbridge-1.0.0/tests/test_loader.py +111 -0
- vernacbridge-1.0.0/tests/test_memory.py +218 -0
- vernacbridge-1.0.0/tests/test_pipeline.py +611 -0
- vernacbridge-1.0.0/tests/test_search.py +66 -0
- vernacbridge-1.0.0/tests/test_translator_emotion_intent.py +97 -0
- vernacbridge-1.0.0/tests/test_utils.py +67 -0
- vernacbridge-1.0.0/tests/test_validator.py +107 -0
- vernacbridge-1.0.0/vernacbridge/__init__.py +217 -0
- vernacbridge-1.0.0/vernacbridge/cleaner.py +107 -0
- vernacbridge-1.0.0/vernacbridge/config.py +154 -0
- vernacbridge-1.0.0/vernacbridge/context.py +132 -0
- vernacbridge-1.0.0/vernacbridge/core.py +158 -0
- vernacbridge-1.0.0/vernacbridge/dataset_engine.py +146 -0
- vernacbridge-1.0.0/vernacbridge/datasets/build_report.json +65 -0
- vernacbridge-1.0.0/vernacbridge/datasets/master_dataset.csv +3776 -0
- vernacbridge-1.0.0/vernacbridge/datasets/vernacbridge_career_1000.csv +1001 -0
- vernacbridge-1.0.0/vernacbridge/datasets/vernacbridge_education_1000.csv +1001 -0
- vernacbridge-1.0.0/vernacbridge/datasets/vernacbridge_family_1000.csv +1001 -0
- vernacbridge-1.0.0/vernacbridge/datasets/vernacbridge_general_1000.csv +1001 -0
- vernacbridge-1.0.0/vernacbridge/emotion.py +68 -0
- vernacbridge-1.0.0/vernacbridge/exceptions.py +67 -0
- vernacbridge-1.0.0/vernacbridge/intelligence.py +127 -0
- vernacbridge-1.0.0/vernacbridge/intent.py +71 -0
- vernacbridge-1.0.0/vernacbridge/loader.py +120 -0
- vernacbridge-1.0.0/vernacbridge/memory.py +165 -0
- vernacbridge-1.0.0/vernacbridge/pipeline.py +434 -0
- vernacbridge-1.0.0/vernacbridge/protocols.py +154 -0
- vernacbridge-1.0.0/vernacbridge/py.typed +0 -0
- vernacbridge-1.0.0/vernacbridge/search.py +82 -0
- vernacbridge-1.0.0/vernacbridge/translator.py +103 -0
- vernacbridge-1.0.0/vernacbridge/utils.py +121 -0
- vernacbridge-1.0.0/vernacbridge/validator.py +134 -0
- vernacbridge-1.0.0/vernacbridge.egg-info/PKG-INFO +217 -0
- vernacbridge-1.0.0/vernacbridge.egg-info/SOURCES.txt +47 -0
- vernacbridge-1.0.0/vernacbridge.egg-info/dependency_links.txt +1 -0
- vernacbridge-1.0.0/vernacbridge.egg-info/entry_points.txt +2 -0
- vernacbridge-1.0.0/vernacbridge.egg-info/requires.txt +9 -0
- vernacbridge-1.0.0/vernacbridge.egg-info/top_level.txt +1 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 VernacBridge Contributors
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,217 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: vernacbridge
|
|
3
|
+
Version: 1.0.0
|
|
4
|
+
Summary: A Python NLP library for understanding Tanglish (Tamil written in English letters).
|
|
5
|
+
Author: VernacBridge Contributors
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/santhiyaviswanth007-lang/vernacbridge
|
|
8
|
+
Project-URL: Repository, https://github.com/santhiyaviswanth007-lang/vernacbridge
|
|
9
|
+
Project-URL: Issues, https://github.com/santhiyaviswanth007-lang/vernacbridge/issues
|
|
10
|
+
Keywords: tanglish,tamil,nlp,translation,emotion-detection,intent-detection
|
|
11
|
+
Classifier: Development Status :: 3 - Alpha
|
|
12
|
+
Classifier: Intended Audience :: Developers
|
|
13
|
+
Classifier: Programming Language :: Python :: 3
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
16
|
+
Classifier: Topic :: Text Processing :: Linguistic
|
|
17
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
18
|
+
Requires-Python: >=3.11
|
|
19
|
+
Description-Content-Type: text/markdown
|
|
20
|
+
License-File: LICENSE
|
|
21
|
+
Requires-Dist: pandas>=2.0
|
|
22
|
+
Requires-Dist: rapidfuzz>=3.0
|
|
23
|
+
Provides-Extra: dev
|
|
24
|
+
Requires-Dist: pytest>=7.0; extra == "dev"
|
|
25
|
+
Requires-Dist: pytest-cov>=4.0; extra == "dev"
|
|
26
|
+
Requires-Dist: black>=24.0; extra == "dev"
|
|
27
|
+
Requires-Dist: ruff>=0.4; extra == "dev"
|
|
28
|
+
Requires-Dist: mypy>=1.8; extra == "dev"
|
|
29
|
+
Dynamic: license-file
|
|
30
|
+
|
|
31
|
+
# VernacBridge
|
|
32
|
+
|
|
33
|
+
A Python library that helps Tamil-speaking people express what they mean in
|
|
34
|
+
**Tanglish** (Tamil written in English letters) and get it back in clear
|
|
35
|
+
English — with intent, emotion, and confidence preserved, not just a literal
|
|
36
|
+
word-for-word translation.
|
|
37
|
+
|
|
38
|
+
The goal isn't just "translate text." It's to help someone who knows
|
|
39
|
+
exactly what they want to say, but feels less confident saying it in
|
|
40
|
+
English, communicate it anyway — for messages, notes, and everyday
|
|
41
|
+
technical or personal communication — without losing what they meant or
|
|
42
|
+
how they felt saying it.
|
|
43
|
+
|
|
44
|
+
## Status — V1, Phases 1–4 complete
|
|
45
|
+
|
|
46
|
+
| Phase | What it adds |
|
|
47
|
+
|---|---|
|
|
48
|
+
| 1 — Translation Pipeline | Normalize → match against the corpus → structured result: translation, intent, emotion, keywords, urgency, confidence. Honest about uncertainty: below-threshold matches return the closest candidates instead of a guess. |
|
|
49
|
+
| 2 — Dataset Intelligence | Opt-in (`explain=True`) transparency into *why* a match won: runner-up, score margin, full ranking. Zero effect on the default output or performance when unused. |
|
|
50
|
+
| 3 — Memory Layer | Opt-in, session-scoped conversation history (`MemoryEngine`), short-term (RAM) and long-term (disk) storage. Nothing is recorded unless you explicitly inject it. |
|
|
51
|
+
| 4 — Context Engine | Resolves short follow-ups ("Why?") against the most recent confident turn in the same session, using Memory. Mechanical (word-count + prior-turn lookup), not a language model. |
|
|
52
|
+
|
|
53
|
+
Verified against a real, human-labelled ~4,000-row Tanglish corpus spanning
|
|
54
|
+
career, education, family, and general-conversation domains. 209 automated
|
|
55
|
+
tests.
|
|
56
|
+
|
|
57
|
+
Every phase beyond 1 is **inert by default** — a caller who just does
|
|
58
|
+
`vernacbridge.translate(...)` gets exactly Phase 1's behavior. Nothing is
|
|
59
|
+
enabled, recorded, or computed unless you ask for it.
|
|
60
|
+
|
|
61
|
+
## Installation
|
|
62
|
+
|
|
63
|
+
```bash
|
|
64
|
+
git clone https://github.com/santhiyaviswanth007-lang/vernacbridge.git
|
|
65
|
+
cd vernacbridge
|
|
66
|
+
pip install .
|
|
67
|
+
```
|
|
68
|
+
|
|
69
|
+
That's a genuine, non-editable install — `datasets/master_dataset.csv` ships
|
|
70
|
+
as real package data, so `import vernacbridge` works from anywhere, not
|
|
71
|
+
just from inside this repo.
|
|
72
|
+
|
|
73
|
+
For development (editable install, plus tests and linting):
|
|
74
|
+
|
|
75
|
+
```bash
|
|
76
|
+
pip install -e ".[dev]"
|
|
77
|
+
```
|
|
78
|
+
|
|
79
|
+
`requirements.txt` lists the same runtime dependencies directly, if you'd
|
|
80
|
+
rather manage them yourself: `pip install -r requirements.txt`.
|
|
81
|
+
|
|
82
|
+
RapidFuzz is used for similarity scoring; if it isn't installed, VernacBridge
|
|
83
|
+
falls back to `difflib` with a logged warning (lower accuracy, same API).
|
|
84
|
+
|
|
85
|
+
## Quick start
|
|
86
|
+
|
|
87
|
+
```python
|
|
88
|
+
import vernacbridge
|
|
89
|
+
|
|
90
|
+
vernacbridge.translate("enaku bayama iruku")
|
|
91
|
+
vernacbridge.detect_emotion("enaku bayama iruku")
|
|
92
|
+
vernacbridge.detect_intent("enaku bayama iruku")
|
|
93
|
+
vernacbridge.search_by_emotion("fear")
|
|
94
|
+
```
|
|
95
|
+
|
|
96
|
+
### The full structured pipeline
|
|
97
|
+
|
|
98
|
+
```python
|
|
99
|
+
vernacbridge.translate_pipeline("enaku bayama iruku")
|
|
100
|
+
```
|
|
101
|
+
|
|
102
|
+
Returns translation, intent, emotion, keywords, urgency, confidence, the
|
|
103
|
+
matched corpus sentence, and — if confidence is too low to commit to one
|
|
104
|
+
answer — the closest candidates instead of a wrong guess.
|
|
105
|
+
|
|
106
|
+
### Explain mode (Phase 2) — why a match won
|
|
107
|
+
|
|
108
|
+
```python
|
|
109
|
+
vernacbridge.translate_pipeline("enaku bayama iruku", explain=True)
|
|
110
|
+
```
|
|
111
|
+
|
|
112
|
+
Adds an `"explanation"` key: the runner-up candidate, the score margin, and
|
|
113
|
+
whether it was a close call. Absent entirely unless you ask for it.
|
|
114
|
+
|
|
115
|
+
### Memory and Context (Phases 3–4) — conversation continuity
|
|
116
|
+
|
|
117
|
+
```python
|
|
118
|
+
from vernacbridge import VernacBridge, MemoryEngine, ContextEngine
|
|
119
|
+
|
|
120
|
+
memory = MemoryEngine()
|
|
121
|
+
context = ContextEngine(memory_engine=memory) # must share the SAME memory instance
|
|
122
|
+
vb = VernacBridge(memory_engine=memory, context_engine=context).load()
|
|
123
|
+
|
|
124
|
+
vb.translate_pipeline("enaku bayama iruku", session_id="user-1")
|
|
125
|
+
vb.translate_pipeline("yean?", session_id="user-1") # "why?" -- resolved using the prior turn
|
|
126
|
+
|
|
127
|
+
memory.recall("user-1") # full turn history for this session
|
|
128
|
+
```
|
|
129
|
+
|
|
130
|
+
`ContextEngine` requires `memory_engine` explicitly, and it must be the
|
|
131
|
+
*same instance* given to `VernacBridge`/`TranslationPipeline` — this is
|
|
132
|
+
enforced (it raises if omitted), not just documented, so context resolution
|
|
133
|
+
can't silently read from an empty, disconnected memory store.
|
|
134
|
+
|
|
135
|
+
### Custom dataset path / multiple independent instances
|
|
136
|
+
|
|
137
|
+
```python
|
|
138
|
+
from vernacbridge import VernacBridge
|
|
139
|
+
|
|
140
|
+
vb = VernacBridge(master_dataset_path="path/to/master_dataset.csv").load()
|
|
141
|
+
vb.translate("enaku bayama iruku")
|
|
142
|
+
```
|
|
143
|
+
|
|
144
|
+
See `examples/full_pipeline_demo.py` for all of the above run together
|
|
145
|
+
against the real corpus.
|
|
146
|
+
|
|
147
|
+
## Building the master dataset
|
|
148
|
+
|
|
149
|
+
Raw, per-domain CSVs live in `vernacbridge/datasets/` (columns:
|
|
150
|
+
`tanglish_input`, `english_output`, `intent`, `emotion`, `keywords`,
|
|
151
|
+
`urgency`, `confidence`). Adding a new domain requires only dropping a new
|
|
152
|
+
CSV into that directory — no code changes.
|
|
153
|
+
|
|
154
|
+
```bash
|
|
155
|
+
python -m vernacbridge.dataset_engine
|
|
156
|
+
# or, once installed:
|
|
157
|
+
vernacbridge-build-dataset
|
|
158
|
+
```
|
|
159
|
+
|
|
160
|
+
This runs the full pipeline — load, validate, merge, deduplicate — and
|
|
161
|
+
writes `vernacbridge/datasets/master_dataset.csv` plus
|
|
162
|
+
`vernacbridge/datasets/build_report.json`.
|
|
163
|
+
|
|
164
|
+
## Architecture
|
|
165
|
+
|
|
166
|
+
```
|
|
167
|
+
vernacbridge/
|
|
168
|
+
__init__.py # VernacBridge facade + module-level convenience functions
|
|
169
|
+
config.py # Single source of truth: schema, label vocab, thresholds
|
|
170
|
+
exceptions.py # Typed exception hierarchy
|
|
171
|
+
loader.py # CSV discovery + raw loading (Dataset Engine)
|
|
172
|
+
validator.py # Column/label/empty-row validation (Dataset Engine)
|
|
173
|
+
cleaner.py # Exact + RapidFuzz near-duplicate removal (Dataset Engine)
|
|
174
|
+
dataset_engine.py # Orchestrates loader -> validator -> cleaner -> master_dataset.csv
|
|
175
|
+
core.py # MatchEngine: shared corpus + nearest-neighbor matching
|
|
176
|
+
translator.py # Translation via MatchEngine
|
|
177
|
+
emotion.py # Emotion detection via MatchEngine
|
|
178
|
+
intent.py # Intent detection via MatchEngine
|
|
179
|
+
search.py # Structured search (by emotion/intent/keyword)
|
|
180
|
+
pipeline.py # TranslationPipeline: the structured, extensible orchestrator
|
|
181
|
+
intelligence.py # DatasetIntelligence: opt-in match explanations (Phase 2)
|
|
182
|
+
memory.py # MemoryEngine + stores: opt-in conversation history (Phase 3)
|
|
183
|
+
context.py # ContextEngine: mechanical follow-up resolution (Phase 4)
|
|
184
|
+
protocols.py # Structural (typing.Protocol) contracts for every
|
|
185
|
+
# injectable collaborator above -- no inheritance required
|
|
186
|
+
datasets/ # Raw per-domain CSVs + generated master_dataset.csv
|
|
187
|
+
# (packaged as real package data -- ships with the
|
|
188
|
+
# installed package, not just the source checkout)
|
|
189
|
+
tests/ # pytest suite (unit + real-dataset integration)
|
|
190
|
+
examples/ # Runnable usage examples
|
|
191
|
+
docs/ # Per-phase design docs + FUTURE_VISION.md
|
|
192
|
+
```
|
|
193
|
+
|
|
194
|
+
Every runtime engine is built on the same `MatchEngine`, so upgrading the
|
|
195
|
+
matching strategy only requires changing `core.py`. Every optional
|
|
196
|
+
capability beyond Phase 1 (`context_engine`, `memory_engine`,
|
|
197
|
+
`plugin_manager`, `conversation_engine`, `dataset_intelligence`) is
|
|
198
|
+
dependency-injected via a `Protocol` in `protocols.py` — any object with
|
|
199
|
+
matching methods works, no base class required.
|
|
200
|
+
|
|
201
|
+
## Running tests
|
|
202
|
+
|
|
203
|
+
```bash
|
|
204
|
+
pytest
|
|
205
|
+
```
|
|
206
|
+
|
|
207
|
+
Unit tests use small, disposable fixture datasets. A separate integration
|
|
208
|
+
suite (`tests/test_integration_real_dataset.py`) runs against the real
|
|
209
|
+
`vernacbridge/datasets/master_dataset.csv` and is skipped automatically if
|
|
210
|
+
that file hasn't been built yet.
|
|
211
|
+
|
|
212
|
+
## What's next
|
|
213
|
+
|
|
214
|
+
V1's scope is intentionally Tanglish-only, built on the existing corpus —
|
|
215
|
+
see `docs/FUTURE_VISION.md` for ideas that are explicitly deferred (a
|
|
216
|
+
professional-communication data domain, multilingual expansion, and other
|
|
217
|
+
larger directions) rather than accidentally becoming V1 requirements.
|
|
@@ -0,0 +1,187 @@
|
|
|
1
|
+
# VernacBridge
|
|
2
|
+
|
|
3
|
+
A Python library that helps Tamil-speaking people express what they mean in
|
|
4
|
+
**Tanglish** (Tamil written in English letters) and get it back in clear
|
|
5
|
+
English — with intent, emotion, and confidence preserved, not just a literal
|
|
6
|
+
word-for-word translation.
|
|
7
|
+
|
|
8
|
+
The goal isn't just "translate text." It's to help someone who knows
|
|
9
|
+
exactly what they want to say, but feels less confident saying it in
|
|
10
|
+
English, communicate it anyway — for messages, notes, and everyday
|
|
11
|
+
technical or personal communication — without losing what they meant or
|
|
12
|
+
how they felt saying it.
|
|
13
|
+
|
|
14
|
+
## Status — V1, Phases 1–4 complete
|
|
15
|
+
|
|
16
|
+
| Phase | What it adds |
|
|
17
|
+
|---|---|
|
|
18
|
+
| 1 — Translation Pipeline | Normalize → match against the corpus → structured result: translation, intent, emotion, keywords, urgency, confidence. Honest about uncertainty: below-threshold matches return the closest candidates instead of a guess. |
|
|
19
|
+
| 2 — Dataset Intelligence | Opt-in (`explain=True`) transparency into *why* a match won: runner-up, score margin, full ranking. Zero effect on the default output or performance when unused. |
|
|
20
|
+
| 3 — Memory Layer | Opt-in, session-scoped conversation history (`MemoryEngine`), short-term (RAM) and long-term (disk) storage. Nothing is recorded unless you explicitly inject it. |
|
|
21
|
+
| 4 — Context Engine | Resolves short follow-ups ("Why?") against the most recent confident turn in the same session, using Memory. Mechanical (word-count + prior-turn lookup), not a language model. |
|
|
22
|
+
|
|
23
|
+
Verified against a real, human-labelled ~4,000-row Tanglish corpus spanning
|
|
24
|
+
career, education, family, and general-conversation domains. 209 automated
|
|
25
|
+
tests.
|
|
26
|
+
|
|
27
|
+
Every phase beyond 1 is **inert by default** — a caller who just does
|
|
28
|
+
`vernacbridge.translate(...)` gets exactly Phase 1's behavior. Nothing is
|
|
29
|
+
enabled, recorded, or computed unless you ask for it.
|
|
30
|
+
|
|
31
|
+
## Installation
|
|
32
|
+
|
|
33
|
+
```bash
|
|
34
|
+
git clone https://github.com/santhiyaviswanth007-lang/vernacbridge.git
|
|
35
|
+
cd vernacbridge
|
|
36
|
+
pip install .
|
|
37
|
+
```
|
|
38
|
+
|
|
39
|
+
That's a genuine, non-editable install — `datasets/master_dataset.csv` ships
|
|
40
|
+
as real package data, so `import vernacbridge` works from anywhere, not
|
|
41
|
+
just from inside this repo.
|
|
42
|
+
|
|
43
|
+
For development (editable install, plus tests and linting):
|
|
44
|
+
|
|
45
|
+
```bash
|
|
46
|
+
pip install -e ".[dev]"
|
|
47
|
+
```
|
|
48
|
+
|
|
49
|
+
`requirements.txt` lists the same runtime dependencies directly, if you'd
|
|
50
|
+
rather manage them yourself: `pip install -r requirements.txt`.
|
|
51
|
+
|
|
52
|
+
RapidFuzz is used for similarity scoring; if it isn't installed, VernacBridge
|
|
53
|
+
falls back to `difflib` with a logged warning (lower accuracy, same API).
|
|
54
|
+
|
|
55
|
+
## Quick start
|
|
56
|
+
|
|
57
|
+
```python
|
|
58
|
+
import vernacbridge
|
|
59
|
+
|
|
60
|
+
vernacbridge.translate("enaku bayama iruku")
|
|
61
|
+
vernacbridge.detect_emotion("enaku bayama iruku")
|
|
62
|
+
vernacbridge.detect_intent("enaku bayama iruku")
|
|
63
|
+
vernacbridge.search_by_emotion("fear")
|
|
64
|
+
```
|
|
65
|
+
|
|
66
|
+
### The full structured pipeline
|
|
67
|
+
|
|
68
|
+
```python
|
|
69
|
+
vernacbridge.translate_pipeline("enaku bayama iruku")
|
|
70
|
+
```
|
|
71
|
+
|
|
72
|
+
Returns translation, intent, emotion, keywords, urgency, confidence, the
|
|
73
|
+
matched corpus sentence, and — if confidence is too low to commit to one
|
|
74
|
+
answer — the closest candidates instead of a wrong guess.
|
|
75
|
+
|
|
76
|
+
### Explain mode (Phase 2) — why a match won
|
|
77
|
+
|
|
78
|
+
```python
|
|
79
|
+
vernacbridge.translate_pipeline("enaku bayama iruku", explain=True)
|
|
80
|
+
```
|
|
81
|
+
|
|
82
|
+
Adds an `"explanation"` key: the runner-up candidate, the score margin, and
|
|
83
|
+
whether it was a close call. Absent entirely unless you ask for it.
|
|
84
|
+
|
|
85
|
+
### Memory and Context (Phases 3–4) — conversation continuity
|
|
86
|
+
|
|
87
|
+
```python
|
|
88
|
+
from vernacbridge import VernacBridge, MemoryEngine, ContextEngine
|
|
89
|
+
|
|
90
|
+
memory = MemoryEngine()
|
|
91
|
+
context = ContextEngine(memory_engine=memory) # must share the SAME memory instance
|
|
92
|
+
vb = VernacBridge(memory_engine=memory, context_engine=context).load()
|
|
93
|
+
|
|
94
|
+
vb.translate_pipeline("enaku bayama iruku", session_id="user-1")
|
|
95
|
+
vb.translate_pipeline("yean?", session_id="user-1") # "why?" -- resolved using the prior turn
|
|
96
|
+
|
|
97
|
+
memory.recall("user-1") # full turn history for this session
|
|
98
|
+
```
|
|
99
|
+
|
|
100
|
+
`ContextEngine` requires `memory_engine` explicitly, and it must be the
|
|
101
|
+
*same instance* given to `VernacBridge`/`TranslationPipeline` — this is
|
|
102
|
+
enforced (it raises if omitted), not just documented, so context resolution
|
|
103
|
+
can't silently read from an empty, disconnected memory store.
|
|
104
|
+
|
|
105
|
+
### Custom dataset path / multiple independent instances
|
|
106
|
+
|
|
107
|
+
```python
|
|
108
|
+
from vernacbridge import VernacBridge
|
|
109
|
+
|
|
110
|
+
vb = VernacBridge(master_dataset_path="path/to/master_dataset.csv").load()
|
|
111
|
+
vb.translate("enaku bayama iruku")
|
|
112
|
+
```
|
|
113
|
+
|
|
114
|
+
See `examples/full_pipeline_demo.py` for all of the above run together
|
|
115
|
+
against the real corpus.
|
|
116
|
+
|
|
117
|
+
## Building the master dataset
|
|
118
|
+
|
|
119
|
+
Raw, per-domain CSVs live in `vernacbridge/datasets/` (columns:
|
|
120
|
+
`tanglish_input`, `english_output`, `intent`, `emotion`, `keywords`,
|
|
121
|
+
`urgency`, `confidence`). Adding a new domain requires only dropping a new
|
|
122
|
+
CSV into that directory — no code changes.
|
|
123
|
+
|
|
124
|
+
```bash
|
|
125
|
+
python -m vernacbridge.dataset_engine
|
|
126
|
+
# or, once installed:
|
|
127
|
+
vernacbridge-build-dataset
|
|
128
|
+
```
|
|
129
|
+
|
|
130
|
+
This runs the full pipeline — load, validate, merge, deduplicate — and
|
|
131
|
+
writes `vernacbridge/datasets/master_dataset.csv` plus
|
|
132
|
+
`vernacbridge/datasets/build_report.json`.
|
|
133
|
+
|
|
134
|
+
## Architecture
|
|
135
|
+
|
|
136
|
+
```
|
|
137
|
+
vernacbridge/
|
|
138
|
+
__init__.py # VernacBridge facade + module-level convenience functions
|
|
139
|
+
config.py # Single source of truth: schema, label vocab, thresholds
|
|
140
|
+
exceptions.py # Typed exception hierarchy
|
|
141
|
+
loader.py # CSV discovery + raw loading (Dataset Engine)
|
|
142
|
+
validator.py # Column/label/empty-row validation (Dataset Engine)
|
|
143
|
+
cleaner.py # Exact + RapidFuzz near-duplicate removal (Dataset Engine)
|
|
144
|
+
dataset_engine.py # Orchestrates loader -> validator -> cleaner -> master_dataset.csv
|
|
145
|
+
core.py # MatchEngine: shared corpus + nearest-neighbor matching
|
|
146
|
+
translator.py # Translation via MatchEngine
|
|
147
|
+
emotion.py # Emotion detection via MatchEngine
|
|
148
|
+
intent.py # Intent detection via MatchEngine
|
|
149
|
+
search.py # Structured search (by emotion/intent/keyword)
|
|
150
|
+
pipeline.py # TranslationPipeline: the structured, extensible orchestrator
|
|
151
|
+
intelligence.py # DatasetIntelligence: opt-in match explanations (Phase 2)
|
|
152
|
+
memory.py # MemoryEngine + stores: opt-in conversation history (Phase 3)
|
|
153
|
+
context.py # ContextEngine: mechanical follow-up resolution (Phase 4)
|
|
154
|
+
protocols.py # Structural (typing.Protocol) contracts for every
|
|
155
|
+
# injectable collaborator above -- no inheritance required
|
|
156
|
+
datasets/ # Raw per-domain CSVs + generated master_dataset.csv
|
|
157
|
+
# (packaged as real package data -- ships with the
|
|
158
|
+
# installed package, not just the source checkout)
|
|
159
|
+
tests/ # pytest suite (unit + real-dataset integration)
|
|
160
|
+
examples/ # Runnable usage examples
|
|
161
|
+
docs/ # Per-phase design docs + FUTURE_VISION.md
|
|
162
|
+
```
|
|
163
|
+
|
|
164
|
+
Every runtime engine is built on the same `MatchEngine`, so upgrading the
|
|
165
|
+
matching strategy only requires changing `core.py`. Every optional
|
|
166
|
+
capability beyond Phase 1 (`context_engine`, `memory_engine`,
|
|
167
|
+
`plugin_manager`, `conversation_engine`, `dataset_intelligence`) is
|
|
168
|
+
dependency-injected via a `Protocol` in `protocols.py` — any object with
|
|
169
|
+
matching methods works, no base class required.
|
|
170
|
+
|
|
171
|
+
## Running tests
|
|
172
|
+
|
|
173
|
+
```bash
|
|
174
|
+
pytest
|
|
175
|
+
```
|
|
176
|
+
|
|
177
|
+
Unit tests use small, disposable fixture datasets. A separate integration
|
|
178
|
+
suite (`tests/test_integration_real_dataset.py`) runs against the real
|
|
179
|
+
`vernacbridge/datasets/master_dataset.csv` and is skipped automatically if
|
|
180
|
+
that file hasn't been built yet.
|
|
181
|
+
|
|
182
|
+
## What's next
|
|
183
|
+
|
|
184
|
+
V1's scope is intentionally Tanglish-only, built on the existing corpus —
|
|
185
|
+
see `docs/FUTURE_VISION.md` for ideas that are explicitly deferred (a
|
|
186
|
+
professional-communication data domain, multilingual expansion, and other
|
|
187
|
+
larger directions) rather than accidentally becoming V1 requirements.
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=68", "wheel"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "vernacbridge"
|
|
7
|
+
version = "1.0.0"
|
|
8
|
+
description = "A Python NLP library for understanding Tanglish (Tamil written in English letters)."
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.11"
|
|
11
|
+
license = "MIT"
|
|
12
|
+
license-files = ["LICENSE"]
|
|
13
|
+
authors = [
|
|
14
|
+
{ name = "VernacBridge Contributors" }
|
|
15
|
+
]
|
|
16
|
+
keywords = ["tanglish", "tamil", "nlp", "translation", "emotion-detection", "intent-detection"]
|
|
17
|
+
classifiers = [
|
|
18
|
+
"Development Status :: 3 - Alpha",
|
|
19
|
+
"Intended Audience :: Developers",
|
|
20
|
+
"Programming Language :: Python :: 3",
|
|
21
|
+
"Programming Language :: Python :: 3.11",
|
|
22
|
+
"Programming Language :: Python :: 3.12",
|
|
23
|
+
"Topic :: Text Processing :: Linguistic",
|
|
24
|
+
"Topic :: Scientific/Engineering :: Artificial Intelligence",
|
|
25
|
+
]
|
|
26
|
+
dependencies = [
|
|
27
|
+
"pandas>=2.0",
|
|
28
|
+
"rapidfuzz>=3.0",
|
|
29
|
+
]
|
|
30
|
+
|
|
31
|
+
[project.optional-dependencies]
|
|
32
|
+
dev = [
|
|
33
|
+
"pytest>=7.0",
|
|
34
|
+
"pytest-cov>=4.0",
|
|
35
|
+
"black>=24.0",
|
|
36
|
+
"ruff>=0.4",
|
|
37
|
+
"mypy>=1.8",
|
|
38
|
+
]
|
|
39
|
+
|
|
40
|
+
[project.urls]
|
|
41
|
+
Homepage = "https://github.com/santhiyaviswanth007-lang/vernacbridge"
|
|
42
|
+
Repository = "https://github.com/santhiyaviswanth007-lang/vernacbridge"
|
|
43
|
+
Issues = "https://github.com/santhiyaviswanth007-lang/vernacbridge/issues"
|
|
44
|
+
|
|
45
|
+
[project.scripts]
|
|
46
|
+
vernacbridge-build-dataset = "vernacbridge.dataset_engine:main"
|
|
47
|
+
|
|
48
|
+
[tool.setuptools.packages.find]
|
|
49
|
+
include = ["vernacbridge*"]
|
|
50
|
+
|
|
51
|
+
[tool.setuptools.package-data]
|
|
52
|
+
vernacbridge = ["py.typed", "datasets/*.csv", "datasets/*.json"]
|
|
53
|
+
|
|
54
|
+
[tool.pytest.ini_options]
|
|
55
|
+
testpaths = ["tests"]
|
|
56
|
+
|
|
57
|
+
[tool.ruff]
|
|
58
|
+
line-length = 100
|
|
59
|
+
target-version = "py311"
|
|
60
|
+
|
|
61
|
+
[tool.black]
|
|
62
|
+
line-length = 100
|
|
63
|
+
target-version = ["py311"]
|
|
64
|
+
|
|
65
|
+
[tool.mypy]
|
|
66
|
+
python_version = "3.11"
|
|
67
|
+
ignore_missing_imports = true
|
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
"""Tests for vernacbridge.cleaner.DatasetCleaner."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import pandas as pd
|
|
6
|
+
import pytest
|
|
7
|
+
|
|
8
|
+
from vernacbridge.cleaner import DatasetCleaner
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
@pytest.fixture
|
|
12
|
+
def cleaner() -> DatasetCleaner:
|
|
13
|
+
return DatasetCleaner(near_duplicate_threshold=92.0)
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def _df(rows: list[dict]) -> pd.DataFrame:
|
|
17
|
+
return pd.DataFrame(rows)
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
class TestExactDuplicates:
|
|
21
|
+
def test_removes_byte_identical_rows(self, cleaner, clean_rows):
|
|
22
|
+
rows = list(clean_rows) + [dict(clean_rows[0])]
|
|
23
|
+
df, removed = cleaner.remove_exact_duplicates(_df(rows))
|
|
24
|
+
assert removed == 1
|
|
25
|
+
assert len(df) == len(clean_rows)
|
|
26
|
+
|
|
27
|
+
def test_removes_case_and_punctuation_variant_duplicates(self, cleaner, clean_rows):
|
|
28
|
+
variant = dict(clean_rows[0])
|
|
29
|
+
variant["tanglish_input"] = "Enaku, Bayama Iruku!!" # same normalized text
|
|
30
|
+
df, removed = cleaner.remove_exact_duplicates(_df([clean_rows[0], variant]))
|
|
31
|
+
assert removed == 1
|
|
32
|
+
assert len(df) == 1
|
|
33
|
+
|
|
34
|
+
def test_no_duplicates_removes_nothing(self, cleaner, clean_rows):
|
|
35
|
+
df, removed = cleaner.remove_exact_duplicates(_df(clean_rows))
|
|
36
|
+
assert removed == 0
|
|
37
|
+
assert len(df) == len(clean_rows)
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
class TestNearDuplicates:
|
|
41
|
+
def test_removes_near_duplicate_above_threshold(self, cleaner, clean_rows):
|
|
42
|
+
near_dup = dict(clean_rows[0])
|
|
43
|
+
near_dup["tanglish_input"] = "enaku bayama irukku" # spelling variant, ~97% similar
|
|
44
|
+
df, removed = cleaner.remove_near_duplicates(_df([clean_rows[0], near_dup]))
|
|
45
|
+
assert removed == 1
|
|
46
|
+
assert len(df) == 1
|
|
47
|
+
|
|
48
|
+
def test_keeps_first_occurrence_of_a_cluster(self, cleaner, clean_rows):
|
|
49
|
+
near_dup = dict(clean_rows[0])
|
|
50
|
+
near_dup["tanglish_input"] = "enaku bayama irukku"
|
|
51
|
+
df, removed = cleaner.remove_near_duplicates(_df([clean_rows[0], near_dup]))
|
|
52
|
+
assert removed == 1
|
|
53
|
+
assert len(df) == 1
|
|
54
|
+
assert df.iloc[0]["tanglish_input"] == clean_rows[0]["tanglish_input"]
|
|
55
|
+
|
|
56
|
+
def test_dissimilar_rows_both_kept(self, cleaner, clean_rows):
|
|
57
|
+
df, removed = cleaner.remove_near_duplicates(_df(clean_rows))
|
|
58
|
+
assert removed == 0
|
|
59
|
+
assert len(df) == len(clean_rows)
|
|
60
|
+
|
|
61
|
+
def test_semantically_different_sentence_not_treated_as_duplicate(self, cleaner, clean_rows):
|
|
62
|
+
# Inserting a whole extra word ("konjam" = "a little") changes the
|
|
63
|
+
# meaning of the sentence; it must NOT be flagged as a near-duplicate
|
|
64
|
+
# even though it shares most of its characters with the original.
|
|
65
|
+
different = dict(clean_rows[0])
|
|
66
|
+
different["tanglish_input"] = "enaku konjam bayama iruku"
|
|
67
|
+
df, removed = cleaner.remove_near_duplicates(_df([clean_rows[0], different]))
|
|
68
|
+
assert removed == 0
|
|
69
|
+
assert len(df) == 2
|
|
70
|
+
|
|
71
|
+
def test_threshold_is_respected(self, clean_rows):
|
|
72
|
+
near_dup = dict(clean_rows[0])
|
|
73
|
+
near_dup["tanglish_input"] = "enaku bayama irukku" # ~97% similar
|
|
74
|
+
|
|
75
|
+
strict_cleaner = DatasetCleaner(near_duplicate_threshold=99.9)
|
|
76
|
+
df, removed = strict_cleaner.remove_near_duplicates(_df([clean_rows[0], near_dup]))
|
|
77
|
+
assert removed == 0 # 97% similar clears 92.0 but not a 99.9 threshold
|
|
78
|
+
assert len(df) == 2
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
class TestCleanPipeline:
|
|
82
|
+
def test_full_pipeline_removes_exact_and_near_duplicates(self, cleaner, clean_rows):
|
|
83
|
+
exact_dup = dict(clean_rows[0])
|
|
84
|
+
near_dup = dict(clean_rows[0])
|
|
85
|
+
near_dup["tanglish_input"] = "enaku bayama irukku"
|
|
86
|
+
|
|
87
|
+
rows = list(clean_rows) + [exact_dup, near_dup]
|
|
88
|
+
cleaned, report = cleaner.clean(_df(rows))
|
|
89
|
+
|
|
90
|
+
assert report.rows_in == len(rows)
|
|
91
|
+
assert report.exact_duplicates_removed == 1
|
|
92
|
+
assert report.near_duplicates_removed == 1
|
|
93
|
+
assert report.rows_out == len(clean_rows)
|
|
94
|
+
assert report.total_removed == 2
|
|
95
|
+
|
|
96
|
+
def test_clean_resets_index(self, cleaner, clean_rows):
|
|
97
|
+
cleaned, _ = cleaner.clean(_df(clean_rows))
|
|
98
|
+
assert list(cleaned.index) == list(range(len(cleaned)))
|
|
99
|
+
|
|
100
|
+
def test_empty_dataframe_returns_empty(self, cleaner):
|
|
101
|
+
empty = pd.DataFrame(columns=["tanglish_input", "english_output"])
|
|
102
|
+
cleaned, report = cleaner.clean(empty)
|
|
103
|
+
assert cleaned.empty
|
|
104
|
+
assert report.rows_in == 0
|
|
105
|
+
assert report.rows_out == 0
|