vernacbridge 1.0.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (49) hide show
  1. vernacbridge-1.0.0/LICENSE +21 -0
  2. vernacbridge-1.0.0/PKG-INFO +217 -0
  3. vernacbridge-1.0.0/README.md +187 -0
  4. vernacbridge-1.0.0/pyproject.toml +67 -0
  5. vernacbridge-1.0.0/setup.cfg +4 -0
  6. vernacbridge-1.0.0/tests/test_cleaner.py +105 -0
  7. vernacbridge-1.0.0/tests/test_context.py +256 -0
  8. vernacbridge-1.0.0/tests/test_core.py +73 -0
  9. vernacbridge-1.0.0/tests/test_dataset_engine.py +157 -0
  10. vernacbridge-1.0.0/tests/test_integration_real_dataset.py +131 -0
  11. vernacbridge-1.0.0/tests/test_intelligence.py +145 -0
  12. vernacbridge-1.0.0/tests/test_loader.py +111 -0
  13. vernacbridge-1.0.0/tests/test_memory.py +218 -0
  14. vernacbridge-1.0.0/tests/test_pipeline.py +611 -0
  15. vernacbridge-1.0.0/tests/test_search.py +66 -0
  16. vernacbridge-1.0.0/tests/test_translator_emotion_intent.py +97 -0
  17. vernacbridge-1.0.0/tests/test_utils.py +67 -0
  18. vernacbridge-1.0.0/tests/test_validator.py +107 -0
  19. vernacbridge-1.0.0/vernacbridge/__init__.py +217 -0
  20. vernacbridge-1.0.0/vernacbridge/cleaner.py +107 -0
  21. vernacbridge-1.0.0/vernacbridge/config.py +154 -0
  22. vernacbridge-1.0.0/vernacbridge/context.py +132 -0
  23. vernacbridge-1.0.0/vernacbridge/core.py +158 -0
  24. vernacbridge-1.0.0/vernacbridge/dataset_engine.py +146 -0
  25. vernacbridge-1.0.0/vernacbridge/datasets/build_report.json +65 -0
  26. vernacbridge-1.0.0/vernacbridge/datasets/master_dataset.csv +3776 -0
  27. vernacbridge-1.0.0/vernacbridge/datasets/vernacbridge_career_1000.csv +1001 -0
  28. vernacbridge-1.0.0/vernacbridge/datasets/vernacbridge_education_1000.csv +1001 -0
  29. vernacbridge-1.0.0/vernacbridge/datasets/vernacbridge_family_1000.csv +1001 -0
  30. vernacbridge-1.0.0/vernacbridge/datasets/vernacbridge_general_1000.csv +1001 -0
  31. vernacbridge-1.0.0/vernacbridge/emotion.py +68 -0
  32. vernacbridge-1.0.0/vernacbridge/exceptions.py +67 -0
  33. vernacbridge-1.0.0/vernacbridge/intelligence.py +127 -0
  34. vernacbridge-1.0.0/vernacbridge/intent.py +71 -0
  35. vernacbridge-1.0.0/vernacbridge/loader.py +120 -0
  36. vernacbridge-1.0.0/vernacbridge/memory.py +165 -0
  37. vernacbridge-1.0.0/vernacbridge/pipeline.py +434 -0
  38. vernacbridge-1.0.0/vernacbridge/protocols.py +154 -0
  39. vernacbridge-1.0.0/vernacbridge/py.typed +0 -0
  40. vernacbridge-1.0.0/vernacbridge/search.py +82 -0
  41. vernacbridge-1.0.0/vernacbridge/translator.py +103 -0
  42. vernacbridge-1.0.0/vernacbridge/utils.py +121 -0
  43. vernacbridge-1.0.0/vernacbridge/validator.py +134 -0
  44. vernacbridge-1.0.0/vernacbridge.egg-info/PKG-INFO +217 -0
  45. vernacbridge-1.0.0/vernacbridge.egg-info/SOURCES.txt +47 -0
  46. vernacbridge-1.0.0/vernacbridge.egg-info/dependency_links.txt +1 -0
  47. vernacbridge-1.0.0/vernacbridge.egg-info/entry_points.txt +2 -0
  48. vernacbridge-1.0.0/vernacbridge.egg-info/requires.txt +9 -0
  49. vernacbridge-1.0.0/vernacbridge.egg-info/top_level.txt +1 -0
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 VernacBridge Contributors
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,217 @@
1
+ Metadata-Version: 2.4
2
+ Name: vernacbridge
3
+ Version: 1.0.0
4
+ Summary: A Python NLP library for understanding Tanglish (Tamil written in English letters).
5
+ Author: VernacBridge Contributors
6
+ License-Expression: MIT
7
+ Project-URL: Homepage, https://github.com/santhiyaviswanth007-lang/vernacbridge
8
+ Project-URL: Repository, https://github.com/santhiyaviswanth007-lang/vernacbridge
9
+ Project-URL: Issues, https://github.com/santhiyaviswanth007-lang/vernacbridge/issues
10
+ Keywords: tanglish,tamil,nlp,translation,emotion-detection,intent-detection
11
+ Classifier: Development Status :: 3 - Alpha
12
+ Classifier: Intended Audience :: Developers
13
+ Classifier: Programming Language :: Python :: 3
14
+ Classifier: Programming Language :: Python :: 3.11
15
+ Classifier: Programming Language :: Python :: 3.12
16
+ Classifier: Topic :: Text Processing :: Linguistic
17
+ Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
18
+ Requires-Python: >=3.11
19
+ Description-Content-Type: text/markdown
20
+ License-File: LICENSE
21
+ Requires-Dist: pandas>=2.0
22
+ Requires-Dist: rapidfuzz>=3.0
23
+ Provides-Extra: dev
24
+ Requires-Dist: pytest>=7.0; extra == "dev"
25
+ Requires-Dist: pytest-cov>=4.0; extra == "dev"
26
+ Requires-Dist: black>=24.0; extra == "dev"
27
+ Requires-Dist: ruff>=0.4; extra == "dev"
28
+ Requires-Dist: mypy>=1.8; extra == "dev"
29
+ Dynamic: license-file
30
+
31
+ # VernacBridge
32
+
33
+ A Python library that helps Tamil-speaking people express what they mean in
34
+ **Tanglish** (Tamil written in English letters) and get it back in clear
35
+ English — with intent, emotion, and confidence preserved, not just a literal
36
+ word-for-word translation.
37
+
38
+ The goal isn't just "translate text." It's to help someone who knows
39
+ exactly what they want to say, but feels less confident saying it in
40
+ English, communicate it anyway — for messages, notes, and everyday
41
+ technical or personal communication — without losing what they meant or
42
+ how they felt saying it.
43
+
44
+ ## Status — V1, Phases 1–4 complete
45
+
46
+ | Phase | What it adds |
47
+ |---|---|
48
+ | 1 — Translation Pipeline | Normalize → match against the corpus → structured result: translation, intent, emotion, keywords, urgency, confidence. Honest about uncertainty: below-threshold matches return the closest candidates instead of a guess. |
49
+ | 2 — Dataset Intelligence | Opt-in (`explain=True`) transparency into *why* a match won: runner-up, score margin, full ranking. Zero effect on the default output or performance when unused. |
50
+ | 3 — Memory Layer | Opt-in, session-scoped conversation history (`MemoryEngine`), short-term (RAM) and long-term (disk) storage. Nothing is recorded unless you explicitly inject it. |
51
+ | 4 — Context Engine | Resolves short follow-ups ("Why?") against the most recent confident turn in the same session, using Memory. Mechanical (word-count + prior-turn lookup), not a language model. |
52
+
53
+ Verified against a real, human-labelled ~4,000-row Tanglish corpus spanning
54
+ career, education, family, and general-conversation domains. 209 automated
55
+ tests.
56
+
57
+ Every phase beyond 1 is **inert by default** — a caller who just does
58
+ `vernacbridge.translate(...)` gets exactly Phase 1's behavior. Nothing is
59
+ enabled, recorded, or computed unless you ask for it.
60
+
61
+ ## Installation
62
+
63
+ ```bash
64
+ git clone https://github.com/santhiyaviswanth007-lang/vernacbridge.git
65
+ cd vernacbridge
66
+ pip install .
67
+ ```
68
+
69
+ That's a genuine, non-editable install — `datasets/master_dataset.csv` ships
70
+ as real package data, so `import vernacbridge` works from anywhere, not
71
+ just from inside this repo.
72
+
73
+ For development (editable install, plus tests and linting):
74
+
75
+ ```bash
76
+ pip install -e ".[dev]"
77
+ ```
78
+
79
+ `requirements.txt` lists the same runtime dependencies directly, if you'd
80
+ rather manage them yourself: `pip install -r requirements.txt`.
81
+
82
+ RapidFuzz is used for similarity scoring; if it isn't installed, VernacBridge
83
+ falls back to `difflib` with a logged warning (lower accuracy, same API).
84
+
85
+ ## Quick start
86
+
87
+ ```python
88
+ import vernacbridge
89
+
90
+ vernacbridge.translate("enaku bayama iruku")
91
+ vernacbridge.detect_emotion("enaku bayama iruku")
92
+ vernacbridge.detect_intent("enaku bayama iruku")
93
+ vernacbridge.search_by_emotion("fear")
94
+ ```
95
+
96
+ ### The full structured pipeline
97
+
98
+ ```python
99
+ vernacbridge.translate_pipeline("enaku bayama iruku")
100
+ ```
101
+
102
+ Returns translation, intent, emotion, keywords, urgency, confidence, the
103
+ matched corpus sentence, and — if confidence is too low to commit to one
104
+ answer — the closest candidates instead of a wrong guess.
105
+
106
+ ### Explain mode (Phase 2) — why a match won
107
+
108
+ ```python
109
+ vernacbridge.translate_pipeline("enaku bayama iruku", explain=True)
110
+ ```
111
+
112
+ Adds an `"explanation"` key: the runner-up candidate, the score margin, and
113
+ whether it was a close call. Absent entirely unless you ask for it.
114
+
115
+ ### Memory and Context (Phases 3–4) — conversation continuity
116
+
117
+ ```python
118
+ from vernacbridge import VernacBridge, MemoryEngine, ContextEngine
119
+
120
+ memory = MemoryEngine()
121
+ context = ContextEngine(memory_engine=memory) # must share the SAME memory instance
122
+ vb = VernacBridge(memory_engine=memory, context_engine=context).load()
123
+
124
+ vb.translate_pipeline("enaku bayama iruku", session_id="user-1")
125
+ vb.translate_pipeline("yean?", session_id="user-1") # "why?" -- resolved using the prior turn
126
+
127
+ memory.recall("user-1") # full turn history for this session
128
+ ```
129
+
130
+ `ContextEngine` requires `memory_engine` explicitly, and it must be the
131
+ *same instance* given to `VernacBridge`/`TranslationPipeline` — this is
132
+ enforced (it raises if omitted), not just documented, so context resolution
133
+ can't silently read from an empty, disconnected memory store.
134
+
135
+ ### Custom dataset path / multiple independent instances
136
+
137
+ ```python
138
+ from vernacbridge import VernacBridge
139
+
140
+ vb = VernacBridge(master_dataset_path="path/to/master_dataset.csv").load()
141
+ vb.translate("enaku bayama iruku")
142
+ ```
143
+
144
+ See `examples/full_pipeline_demo.py` for all of the above run together
145
+ against the real corpus.
146
+
147
+ ## Building the master dataset
148
+
149
+ Raw, per-domain CSVs live in `vernacbridge/datasets/` (columns:
150
+ `tanglish_input`, `english_output`, `intent`, `emotion`, `keywords`,
151
+ `urgency`, `confidence`). Adding a new domain requires only dropping a new
152
+ CSV into that directory — no code changes.
153
+
154
+ ```bash
155
+ python -m vernacbridge.dataset_engine
156
+ # or, once installed:
157
+ vernacbridge-build-dataset
158
+ ```
159
+
160
+ This runs the full pipeline — load, validate, merge, deduplicate — and
161
+ writes `vernacbridge/datasets/master_dataset.csv` plus
162
+ `vernacbridge/datasets/build_report.json`.
163
+
164
+ ## Architecture
165
+
166
+ ```
167
+ vernacbridge/
168
+ __init__.py # VernacBridge facade + module-level convenience functions
169
+ config.py # Single source of truth: schema, label vocab, thresholds
170
+ exceptions.py # Typed exception hierarchy
171
+ loader.py # CSV discovery + raw loading (Dataset Engine)
172
+ validator.py # Column/label/empty-row validation (Dataset Engine)
173
+ cleaner.py # Exact + RapidFuzz near-duplicate removal (Dataset Engine)
174
+ dataset_engine.py # Orchestrates loader -> validator -> cleaner -> master_dataset.csv
175
+ core.py # MatchEngine: shared corpus + nearest-neighbor matching
176
+ translator.py # Translation via MatchEngine
177
+ emotion.py # Emotion detection via MatchEngine
178
+ intent.py # Intent detection via MatchEngine
179
+ search.py # Structured search (by emotion/intent/keyword)
180
+ pipeline.py # TranslationPipeline: the structured, extensible orchestrator
181
+ intelligence.py # DatasetIntelligence: opt-in match explanations (Phase 2)
182
+ memory.py # MemoryEngine + stores: opt-in conversation history (Phase 3)
183
+ context.py # ContextEngine: mechanical follow-up resolution (Phase 4)
184
+ protocols.py # Structural (typing.Protocol) contracts for every
185
+ # injectable collaborator above -- no inheritance required
186
+ datasets/ # Raw per-domain CSVs + generated master_dataset.csv
187
+ # (packaged as real package data -- ships with the
188
+ # installed package, not just the source checkout)
189
+ tests/ # pytest suite (unit + real-dataset integration)
190
+ examples/ # Runnable usage examples
191
+ docs/ # Per-phase design docs + FUTURE_VISION.md
192
+ ```
193
+
194
+ Every runtime engine is built on the same `MatchEngine`, so upgrading the
195
+ matching strategy only requires changing `core.py`. Every optional
196
+ capability beyond Phase 1 (`context_engine`, `memory_engine`,
197
+ `plugin_manager`, `conversation_engine`, `dataset_intelligence`) is
198
+ dependency-injected via a `Protocol` in `protocols.py` — any object with
199
+ matching methods works, no base class required.
200
+
201
+ ## Running tests
202
+
203
+ ```bash
204
+ pytest
205
+ ```
206
+
207
+ Unit tests use small, disposable fixture datasets. A separate integration
208
+ suite (`tests/test_integration_real_dataset.py`) runs against the real
209
+ `vernacbridge/datasets/master_dataset.csv` and is skipped automatically if
210
+ that file hasn't been built yet.
211
+
212
+ ## What's next
213
+
214
+ V1's scope is intentionally Tanglish-only, built on the existing corpus —
215
+ see `docs/FUTURE_VISION.md` for ideas that are explicitly deferred (a
216
+ professional-communication data domain, multilingual expansion, and other
217
+ larger directions) rather than accidentally becoming V1 requirements.
@@ -0,0 +1,187 @@
1
+ # VernacBridge
2
+
3
+ A Python library that helps Tamil-speaking people express what they mean in
4
+ **Tanglish** (Tamil written in English letters) and get it back in clear
5
+ English — with intent, emotion, and confidence preserved, not just a literal
6
+ word-for-word translation.
7
+
8
+ The goal isn't just "translate text." It's to help someone who knows
9
+ exactly what they want to say, but feels less confident saying it in
10
+ English, communicate it anyway — for messages, notes, and everyday
11
+ technical or personal communication — without losing what they meant or
12
+ how they felt saying it.
13
+
14
+ ## Status — V1, Phases 1–4 complete
15
+
16
+ | Phase | What it adds |
17
+ |---|---|
18
+ | 1 — Translation Pipeline | Normalize → match against the corpus → structured result: translation, intent, emotion, keywords, urgency, confidence. Honest about uncertainty: below-threshold matches return the closest candidates instead of a guess. |
19
+ | 2 — Dataset Intelligence | Opt-in (`explain=True`) transparency into *why* a match won: runner-up, score margin, full ranking. Zero effect on the default output or performance when unused. |
20
+ | 3 — Memory Layer | Opt-in, session-scoped conversation history (`MemoryEngine`), short-term (RAM) and long-term (disk) storage. Nothing is recorded unless you explicitly inject it. |
21
+ | 4 — Context Engine | Resolves short follow-ups ("Why?") against the most recent confident turn in the same session, using Memory. Mechanical (word-count + prior-turn lookup), not a language model. |
22
+
23
+ Verified against a real, human-labelled ~4,000-row Tanglish corpus spanning
24
+ career, education, family, and general-conversation domains. 209 automated
25
+ tests.
26
+
27
+ Every phase beyond 1 is **inert by default** — a caller who just does
28
+ `vernacbridge.translate(...)` gets exactly Phase 1's behavior. Nothing is
29
+ enabled, recorded, or computed unless you ask for it.
30
+
31
+ ## Installation
32
+
33
+ ```bash
34
+ git clone https://github.com/santhiyaviswanth007-lang/vernacbridge.git
35
+ cd vernacbridge
36
+ pip install .
37
+ ```
38
+
39
+ That's a genuine, non-editable install — `datasets/master_dataset.csv` ships
40
+ as real package data, so `import vernacbridge` works from anywhere, not
41
+ just from inside this repo.
42
+
43
+ For development (editable install, plus tests and linting):
44
+
45
+ ```bash
46
+ pip install -e ".[dev]"
47
+ ```
48
+
49
+ `requirements.txt` lists the same runtime dependencies directly, if you'd
50
+ rather manage them yourself: `pip install -r requirements.txt`.
51
+
52
+ RapidFuzz is used for similarity scoring; if it isn't installed, VernacBridge
53
+ falls back to `difflib` with a logged warning (lower accuracy, same API).
54
+
55
+ ## Quick start
56
+
57
+ ```python
58
+ import vernacbridge
59
+
60
+ vernacbridge.translate("enaku bayama iruku")
61
+ vernacbridge.detect_emotion("enaku bayama iruku")
62
+ vernacbridge.detect_intent("enaku bayama iruku")
63
+ vernacbridge.search_by_emotion("fear")
64
+ ```
65
+
66
+ ### The full structured pipeline
67
+
68
+ ```python
69
+ vernacbridge.translate_pipeline("enaku bayama iruku")
70
+ ```
71
+
72
+ Returns translation, intent, emotion, keywords, urgency, confidence, the
73
+ matched corpus sentence, and — if confidence is too low to commit to one
74
+ answer — the closest candidates instead of a wrong guess.
75
+
76
+ ### Explain mode (Phase 2) — why a match won
77
+
78
+ ```python
79
+ vernacbridge.translate_pipeline("enaku bayama iruku", explain=True)
80
+ ```
81
+
82
+ Adds an `"explanation"` key: the runner-up candidate, the score margin, and
83
+ whether it was a close call. Absent entirely unless you ask for it.
84
+
85
+ ### Memory and Context (Phases 3–4) — conversation continuity
86
+
87
+ ```python
88
+ from vernacbridge import VernacBridge, MemoryEngine, ContextEngine
89
+
90
+ memory = MemoryEngine()
91
+ context = ContextEngine(memory_engine=memory) # must share the SAME memory instance
92
+ vb = VernacBridge(memory_engine=memory, context_engine=context).load()
93
+
94
+ vb.translate_pipeline("enaku bayama iruku", session_id="user-1")
95
+ vb.translate_pipeline("yean?", session_id="user-1") # "why?" -- resolved using the prior turn
96
+
97
+ memory.recall("user-1") # full turn history for this session
98
+ ```
99
+
100
+ `ContextEngine` requires `memory_engine` explicitly, and it must be the
101
+ *same instance* given to `VernacBridge`/`TranslationPipeline` — this is
102
+ enforced (it raises if omitted), not just documented, so context resolution
103
+ can't silently read from an empty, disconnected memory store.
104
+
105
+ ### Custom dataset path / multiple independent instances
106
+
107
+ ```python
108
+ from vernacbridge import VernacBridge
109
+
110
+ vb = VernacBridge(master_dataset_path="path/to/master_dataset.csv").load()
111
+ vb.translate("enaku bayama iruku")
112
+ ```
113
+
114
+ See `examples/full_pipeline_demo.py` for all of the above run together
115
+ against the real corpus.
116
+
117
+ ## Building the master dataset
118
+
119
+ Raw, per-domain CSVs live in `vernacbridge/datasets/` (columns:
120
+ `tanglish_input`, `english_output`, `intent`, `emotion`, `keywords`,
121
+ `urgency`, `confidence`). Adding a new domain requires only dropping a new
122
+ CSV into that directory — no code changes.
123
+
124
+ ```bash
125
+ python -m vernacbridge.dataset_engine
126
+ # or, once installed:
127
+ vernacbridge-build-dataset
128
+ ```
129
+
130
+ This runs the full pipeline — load, validate, merge, deduplicate — and
131
+ writes `vernacbridge/datasets/master_dataset.csv` plus
132
+ `vernacbridge/datasets/build_report.json`.
133
+
134
+ ## Architecture
135
+
136
+ ```
137
+ vernacbridge/
138
+ __init__.py # VernacBridge facade + module-level convenience functions
139
+ config.py # Single source of truth: schema, label vocab, thresholds
140
+ exceptions.py # Typed exception hierarchy
141
+ loader.py # CSV discovery + raw loading (Dataset Engine)
142
+ validator.py # Column/label/empty-row validation (Dataset Engine)
143
+ cleaner.py # Exact + RapidFuzz near-duplicate removal (Dataset Engine)
144
+ dataset_engine.py # Orchestrates loader -> validator -> cleaner -> master_dataset.csv
145
+ core.py # MatchEngine: shared corpus + nearest-neighbor matching
146
+ translator.py # Translation via MatchEngine
147
+ emotion.py # Emotion detection via MatchEngine
148
+ intent.py # Intent detection via MatchEngine
149
+ search.py # Structured search (by emotion/intent/keyword)
150
+ pipeline.py # TranslationPipeline: the structured, extensible orchestrator
151
+ intelligence.py # DatasetIntelligence: opt-in match explanations (Phase 2)
152
+ memory.py # MemoryEngine + stores: opt-in conversation history (Phase 3)
153
+ context.py # ContextEngine: mechanical follow-up resolution (Phase 4)
154
+ protocols.py # Structural (typing.Protocol) contracts for every
155
+ # injectable collaborator above -- no inheritance required
156
+ datasets/ # Raw per-domain CSVs + generated master_dataset.csv
157
+ # (packaged as real package data -- ships with the
158
+ # installed package, not just the source checkout)
159
+ tests/ # pytest suite (unit + real-dataset integration)
160
+ examples/ # Runnable usage examples
161
+ docs/ # Per-phase design docs + FUTURE_VISION.md
162
+ ```
163
+
164
+ Every runtime engine is built on the same `MatchEngine`, so upgrading the
165
+ matching strategy only requires changing `core.py`. Every optional
166
+ capability beyond Phase 1 (`context_engine`, `memory_engine`,
167
+ `plugin_manager`, `conversation_engine`, `dataset_intelligence`) is
168
+ dependency-injected via a `Protocol` in `protocols.py` — any object with
169
+ matching methods works, no base class required.
170
+
171
+ ## Running tests
172
+
173
+ ```bash
174
+ pytest
175
+ ```
176
+
177
+ Unit tests use small, disposable fixture datasets. A separate integration
178
+ suite (`tests/test_integration_real_dataset.py`) runs against the real
179
+ `vernacbridge/datasets/master_dataset.csv` and is skipped automatically if
180
+ that file hasn't been built yet.
181
+
182
+ ## What's next
183
+
184
+ V1's scope is intentionally Tanglish-only, built on the existing corpus —
185
+ see `docs/FUTURE_VISION.md` for ideas that are explicitly deferred (a
186
+ professional-communication data domain, multilingual expansion, and other
187
+ larger directions) rather than accidentally becoming V1 requirements.
@@ -0,0 +1,67 @@
1
+ [build-system]
2
+ requires = ["setuptools>=68", "wheel"]
3
+ build-backend = "setuptools.build_meta"
4
+
5
+ [project]
6
+ name = "vernacbridge"
7
+ version = "1.0.0"
8
+ description = "A Python NLP library for understanding Tanglish (Tamil written in English letters)."
9
+ readme = "README.md"
10
+ requires-python = ">=3.11"
11
+ license = "MIT"
12
+ license-files = ["LICENSE"]
13
+ authors = [
14
+ { name = "VernacBridge Contributors" }
15
+ ]
16
+ keywords = ["tanglish", "tamil", "nlp", "translation", "emotion-detection", "intent-detection"]
17
+ classifiers = [
18
+ "Development Status :: 3 - Alpha",
19
+ "Intended Audience :: Developers",
20
+ "Programming Language :: Python :: 3",
21
+ "Programming Language :: Python :: 3.11",
22
+ "Programming Language :: Python :: 3.12",
23
+ "Topic :: Text Processing :: Linguistic",
24
+ "Topic :: Scientific/Engineering :: Artificial Intelligence",
25
+ ]
26
+ dependencies = [
27
+ "pandas>=2.0",
28
+ "rapidfuzz>=3.0",
29
+ ]
30
+
31
+ [project.optional-dependencies]
32
+ dev = [
33
+ "pytest>=7.0",
34
+ "pytest-cov>=4.0",
35
+ "black>=24.0",
36
+ "ruff>=0.4",
37
+ "mypy>=1.8",
38
+ ]
39
+
40
+ [project.urls]
41
+ Homepage = "https://github.com/santhiyaviswanth007-lang/vernacbridge"
42
+ Repository = "https://github.com/santhiyaviswanth007-lang/vernacbridge"
43
+ Issues = "https://github.com/santhiyaviswanth007-lang/vernacbridge/issues"
44
+
45
+ [project.scripts]
46
+ vernacbridge-build-dataset = "vernacbridge.dataset_engine:main"
47
+
48
+ [tool.setuptools.packages.find]
49
+ include = ["vernacbridge*"]
50
+
51
+ [tool.setuptools.package-data]
52
+ vernacbridge = ["py.typed", "datasets/*.csv", "datasets/*.json"]
53
+
54
+ [tool.pytest.ini_options]
55
+ testpaths = ["tests"]
56
+
57
+ [tool.ruff]
58
+ line-length = 100
59
+ target-version = "py311"
60
+
61
+ [tool.black]
62
+ line-length = 100
63
+ target-version = ["py311"]
64
+
65
+ [tool.mypy]
66
+ python_version = "3.11"
67
+ ignore_missing_imports = true
@@ -0,0 +1,4 @@
1
+ [egg_info]
2
+ tag_build =
3
+ tag_date = 0
4
+
@@ -0,0 +1,105 @@
1
+ """Tests for vernacbridge.cleaner.DatasetCleaner."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import pandas as pd
6
+ import pytest
7
+
8
+ from vernacbridge.cleaner import DatasetCleaner
9
+
10
+
11
+ @pytest.fixture
12
+ def cleaner() -> DatasetCleaner:
13
+ return DatasetCleaner(near_duplicate_threshold=92.0)
14
+
15
+
16
+ def _df(rows: list[dict]) -> pd.DataFrame:
17
+ return pd.DataFrame(rows)
18
+
19
+
20
+ class TestExactDuplicates:
21
+ def test_removes_byte_identical_rows(self, cleaner, clean_rows):
22
+ rows = list(clean_rows) + [dict(clean_rows[0])]
23
+ df, removed = cleaner.remove_exact_duplicates(_df(rows))
24
+ assert removed == 1
25
+ assert len(df) == len(clean_rows)
26
+
27
+ def test_removes_case_and_punctuation_variant_duplicates(self, cleaner, clean_rows):
28
+ variant = dict(clean_rows[0])
29
+ variant["tanglish_input"] = "Enaku, Bayama Iruku!!" # same normalized text
30
+ df, removed = cleaner.remove_exact_duplicates(_df([clean_rows[0], variant]))
31
+ assert removed == 1
32
+ assert len(df) == 1
33
+
34
+ def test_no_duplicates_removes_nothing(self, cleaner, clean_rows):
35
+ df, removed = cleaner.remove_exact_duplicates(_df(clean_rows))
36
+ assert removed == 0
37
+ assert len(df) == len(clean_rows)
38
+
39
+
40
+ class TestNearDuplicates:
41
+ def test_removes_near_duplicate_above_threshold(self, cleaner, clean_rows):
42
+ near_dup = dict(clean_rows[0])
43
+ near_dup["tanglish_input"] = "enaku bayama irukku" # spelling variant, ~97% similar
44
+ df, removed = cleaner.remove_near_duplicates(_df([clean_rows[0], near_dup]))
45
+ assert removed == 1
46
+ assert len(df) == 1
47
+
48
+ def test_keeps_first_occurrence_of_a_cluster(self, cleaner, clean_rows):
49
+ near_dup = dict(clean_rows[0])
50
+ near_dup["tanglish_input"] = "enaku bayama irukku"
51
+ df, removed = cleaner.remove_near_duplicates(_df([clean_rows[0], near_dup]))
52
+ assert removed == 1
53
+ assert len(df) == 1
54
+ assert df.iloc[0]["tanglish_input"] == clean_rows[0]["tanglish_input"]
55
+
56
+ def test_dissimilar_rows_both_kept(self, cleaner, clean_rows):
57
+ df, removed = cleaner.remove_near_duplicates(_df(clean_rows))
58
+ assert removed == 0
59
+ assert len(df) == len(clean_rows)
60
+
61
+ def test_semantically_different_sentence_not_treated_as_duplicate(self, cleaner, clean_rows):
62
+ # Inserting a whole extra word ("konjam" = "a little") changes the
63
+ # meaning of the sentence; it must NOT be flagged as a near-duplicate
64
+ # even though it shares most of its characters with the original.
65
+ different = dict(clean_rows[0])
66
+ different["tanglish_input"] = "enaku konjam bayama iruku"
67
+ df, removed = cleaner.remove_near_duplicates(_df([clean_rows[0], different]))
68
+ assert removed == 0
69
+ assert len(df) == 2
70
+
71
+ def test_threshold_is_respected(self, clean_rows):
72
+ near_dup = dict(clean_rows[0])
73
+ near_dup["tanglish_input"] = "enaku bayama irukku" # ~97% similar
74
+
75
+ strict_cleaner = DatasetCleaner(near_duplicate_threshold=99.9)
76
+ df, removed = strict_cleaner.remove_near_duplicates(_df([clean_rows[0], near_dup]))
77
+ assert removed == 0 # 97% similar clears 92.0 but not a 99.9 threshold
78
+ assert len(df) == 2
79
+
80
+
81
+ class TestCleanPipeline:
82
+ def test_full_pipeline_removes_exact_and_near_duplicates(self, cleaner, clean_rows):
83
+ exact_dup = dict(clean_rows[0])
84
+ near_dup = dict(clean_rows[0])
85
+ near_dup["tanglish_input"] = "enaku bayama irukku"
86
+
87
+ rows = list(clean_rows) + [exact_dup, near_dup]
88
+ cleaned, report = cleaner.clean(_df(rows))
89
+
90
+ assert report.rows_in == len(rows)
91
+ assert report.exact_duplicates_removed == 1
92
+ assert report.near_duplicates_removed == 1
93
+ assert report.rows_out == len(clean_rows)
94
+ assert report.total_removed == 2
95
+
96
+ def test_clean_resets_index(self, cleaner, clean_rows):
97
+ cleaned, _ = cleaner.clean(_df(clean_rows))
98
+ assert list(cleaned.index) == list(range(len(cleaned)))
99
+
100
+ def test_empty_dataframe_returns_empty(self, cleaner):
101
+ empty = pd.DataFrame(columns=["tanglish_input", "english_output"])
102
+ cleaned, report = cleaner.clean(empty)
103
+ assert cleaned.empty
104
+ assert report.rows_in == 0
105
+ assert report.rows_out == 0