vernacbridge 1.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- vernacbridge/__init__.py +217 -0
- vernacbridge/cleaner.py +107 -0
- vernacbridge/config.py +154 -0
- vernacbridge/context.py +132 -0
- vernacbridge/core.py +158 -0
- vernacbridge/dataset_engine.py +146 -0
- vernacbridge/datasets/build_report.json +65 -0
- vernacbridge/datasets/master_dataset.csv +3776 -0
- vernacbridge/datasets/vernacbridge_career_1000.csv +1001 -0
- vernacbridge/datasets/vernacbridge_education_1000.csv +1001 -0
- vernacbridge/datasets/vernacbridge_family_1000.csv +1001 -0
- vernacbridge/datasets/vernacbridge_general_1000.csv +1001 -0
- vernacbridge/emotion.py +68 -0
- vernacbridge/exceptions.py +67 -0
- vernacbridge/intelligence.py +127 -0
- vernacbridge/intent.py +71 -0
- vernacbridge/loader.py +120 -0
- vernacbridge/memory.py +165 -0
- vernacbridge/pipeline.py +434 -0
- vernacbridge/protocols.py +154 -0
- vernacbridge/py.typed +0 -0
- vernacbridge/search.py +82 -0
- vernacbridge/translator.py +103 -0
- vernacbridge/utils.py +121 -0
- vernacbridge/validator.py +134 -0
- vernacbridge-1.0.0.dist-info/METADATA +217 -0
- vernacbridge-1.0.0.dist-info/RECORD +31 -0
- vernacbridge-1.0.0.dist-info/WHEEL +5 -0
- vernacbridge-1.0.0.dist-info/entry_points.txt +2 -0
- vernacbridge-1.0.0.dist-info/licenses/LICENSE +21 -0
- vernacbridge-1.0.0.dist-info/top_level.txt +1 -0
vernacbridge/__init__.py
ADDED
|
@@ -0,0 +1,217 @@
|
|
|
1
|
+
"""
|
|
2
|
+
VernacBridge
|
|
3
|
+
============
|
|
4
|
+
|
|
5
|
+
A Python NLP library for understanding Tanglish (Tamil written in English
|
|
6
|
+
letters) -- translation, emotion detection, intent detection, and corpus
|
|
7
|
+
search, built on a continuously growing, human-labelled dataset.
|
|
8
|
+
|
|
9
|
+
Quick start
|
|
10
|
+
-----------
|
|
11
|
+
::
|
|
12
|
+
|
|
13
|
+
import vernacbridge
|
|
14
|
+
|
|
15
|
+
vernacbridge.translate("Enaku bayama iruku")
|
|
16
|
+
vernacbridge.detect_emotion("Enaku bayama iruku")
|
|
17
|
+
vernacbridge.detect_intent("Enaku bayama iruku")
|
|
18
|
+
vernacbridge.search_by_emotion("fear")
|
|
19
|
+
vernacbridge.translate_pipeline("Enaku bayama iruku")
|
|
20
|
+
|
|
21
|
+
Developers debugging match quality can opt into Phase 2's explanation data
|
|
22
|
+
without changing the default output for everyone else::
|
|
23
|
+
|
|
24
|
+
vernacbridge.translate_pipeline("Enaku bayama iruku", explain=True)
|
|
25
|
+
|
|
26
|
+
To remember conversation turns across calls, inject a Phase 3
|
|
27
|
+
:class:`MemoryEngine` -- it stays opt-in, so nothing is recorded unless
|
|
28
|
+
you ask for it::
|
|
29
|
+
|
|
30
|
+
from vernacbridge import MemoryEngine
|
|
31
|
+
|
|
32
|
+
memory = MemoryEngine()
|
|
33
|
+
vb = VernacBridge(memory_engine=memory)
|
|
34
|
+
vb.translate_pipeline("Enaku bayama iruku", session_id="user-1")
|
|
35
|
+
memory.recall("user-1")
|
|
36
|
+
|
|
37
|
+
For multi-instance or custom-dataset use cases, use the :class:`VernacBridge`
|
|
38
|
+
facade instead of the module-level functions::
|
|
39
|
+
|
|
40
|
+
from vernacbridge import VernacBridge
|
|
41
|
+
|
|
42
|
+
vb = VernacBridge(master_dataset_path="path/to/master_dataset.csv")
|
|
43
|
+
vb.translate("Enaku bayama iruku")
|
|
44
|
+
|
|
45
|
+
To (re)build the master dataset from the raw per-domain CSVs::
|
|
46
|
+
|
|
47
|
+
from vernacbridge import DatasetEngine
|
|
48
|
+
|
|
49
|
+
DatasetEngine().build()
|
|
50
|
+
"""
|
|
51
|
+
|
|
52
|
+
from __future__ import annotations
|
|
53
|
+
|
|
54
|
+
from pathlib import Path
|
|
55
|
+
|
|
56
|
+
from . import config
|
|
57
|
+
from .context import ContextEngine
|
|
58
|
+
from .core import Match, MatchEngine
|
|
59
|
+
from .dataset_engine import BuildReport, DatasetEngine
|
|
60
|
+
from .emotion import EmotionDetector, EmotionResult
|
|
61
|
+
from .emotion import detect_emotion as _module_detect_emotion
|
|
62
|
+
from .intent import IntentDetector, IntentResult
|
|
63
|
+
from .intent import detect_intent as _module_detect_intent
|
|
64
|
+
from .intelligence import DatasetIntelligence, MatchExplanation
|
|
65
|
+
from .memory import InMemoryStore, JSONFileStore, MemoryEngine, MemoryStore
|
|
66
|
+
from .pipeline import MatchCandidate, PipelineResult, TranslationPipeline
|
|
67
|
+
from .pipeline import translate_pipeline as _module_translate_pipeline
|
|
68
|
+
from .protocols import (
|
|
69
|
+
ConversationEngineProtocol,
|
|
70
|
+
ContextEngineProtocol,
|
|
71
|
+
DatasetIntelligenceProtocol,
|
|
72
|
+
MemoryEngineProtocol,
|
|
73
|
+
PluginManagerProtocol,
|
|
74
|
+
)
|
|
75
|
+
from .search import SearchEngine
|
|
76
|
+
from .search import search_by_emotion as _module_search_by_emotion
|
|
77
|
+
from .search import search_by_intent as _module_search_by_intent
|
|
78
|
+
from .search import search_by_keyword as _module_search_by_keyword
|
|
79
|
+
from .translator import Translator, TranslationResult
|
|
80
|
+
from .translator import translate as _module_translate
|
|
81
|
+
|
|
82
|
+
__version__ = "1.0.0"
|
|
83
|
+
|
|
84
|
+
__all__ = [
|
|
85
|
+
"VernacBridge",
|
|
86
|
+
"DatasetEngine",
|
|
87
|
+
"BuildReport",
|
|
88
|
+
"MatchEngine",
|
|
89
|
+
"Match",
|
|
90
|
+
"Translator",
|
|
91
|
+
"TranslationResult",
|
|
92
|
+
"EmotionDetector",
|
|
93
|
+
"EmotionResult",
|
|
94
|
+
"IntentDetector",
|
|
95
|
+
"IntentResult",
|
|
96
|
+
"SearchEngine",
|
|
97
|
+
"TranslationPipeline",
|
|
98
|
+
"PipelineResult",
|
|
99
|
+
"MatchCandidate",
|
|
100
|
+
"DatasetIntelligence",
|
|
101
|
+
"MatchExplanation",
|
|
102
|
+
"MemoryEngine",
|
|
103
|
+
"MemoryStore",
|
|
104
|
+
"InMemoryStore",
|
|
105
|
+
"JSONFileStore",
|
|
106
|
+
"ContextEngine",
|
|
107
|
+
"ContextEngineProtocol",
|
|
108
|
+
"MemoryEngineProtocol",
|
|
109
|
+
"PluginManagerProtocol",
|
|
110
|
+
"ConversationEngineProtocol",
|
|
111
|
+
"DatasetIntelligenceProtocol",
|
|
112
|
+
"translate",
|
|
113
|
+
"detect_emotion",
|
|
114
|
+
"detect_intent",
|
|
115
|
+
"search_by_emotion",
|
|
116
|
+
"search_by_intent",
|
|
117
|
+
"search_by_keyword",
|
|
118
|
+
"translate_pipeline",
|
|
119
|
+
]
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
class VernacBridge:
|
|
123
|
+
"""
|
|
124
|
+
Facade over every VernacBridge engine, sharing a single loaded corpus.
|
|
125
|
+
|
|
126
|
+
Prefer this class over the module-level convenience functions whenever
|
|
127
|
+
you need a non-default dataset path, multiple independent instances
|
|
128
|
+
(e.g. in a multi-tenant service), or explicit control over when the
|
|
129
|
+
corpus is loaded into memory.
|
|
130
|
+
"""
|
|
131
|
+
|
|
132
|
+
def __init__(
|
|
133
|
+
self,
|
|
134
|
+
master_dataset_path: str | Path = config.DEFAULT_MASTER_DATASET_PATH,
|
|
135
|
+
*,
|
|
136
|
+
context_engine: ContextEngineProtocol | None = None,
|
|
137
|
+
memory_engine: MemoryEngineProtocol | None = None,
|
|
138
|
+
plugin_manager: PluginManagerProtocol | None = None,
|
|
139
|
+
conversation_engine: ConversationEngineProtocol | None = None,
|
|
140
|
+
dataset_intelligence: DatasetIntelligenceProtocol | None = None,
|
|
141
|
+
) -> None:
|
|
142
|
+
self.engine = MatchEngine(master_dataset_path=master_dataset_path)
|
|
143
|
+
self.translator = Translator(engine=self.engine)
|
|
144
|
+
self.emotion_detector = EmotionDetector(engine=self.engine)
|
|
145
|
+
self.intent_detector = IntentDetector(engine=self.engine)
|
|
146
|
+
self.search_engine = SearchEngine(engine=self.engine)
|
|
147
|
+
self.pipeline = TranslationPipeline(
|
|
148
|
+
engine=self.engine,
|
|
149
|
+
context_engine=context_engine,
|
|
150
|
+
memory_engine=memory_engine,
|
|
151
|
+
plugin_manager=plugin_manager,
|
|
152
|
+
conversation_engine=conversation_engine,
|
|
153
|
+
dataset_intelligence=dataset_intelligence,
|
|
154
|
+
)
|
|
155
|
+
|
|
156
|
+
def load(self) -> "VernacBridge":
|
|
157
|
+
"""Eagerly load the corpus (otherwise it lazy-loads on first use)."""
|
|
158
|
+
self.engine.load()
|
|
159
|
+
return self
|
|
160
|
+
|
|
161
|
+
def translate(self, text: str, min_confidence: float = config.MATCH_CONFIDENCE_THRESHOLD) -> TranslationResult:
|
|
162
|
+
return self.translator.translate(text, min_confidence=min_confidence)
|
|
163
|
+
|
|
164
|
+
def detect_emotion(self, text: str, min_confidence: float = config.MATCH_CONFIDENCE_THRESHOLD) -> EmotionResult:
|
|
165
|
+
return self.emotion_detector.detect_emotion(text, min_confidence=min_confidence)
|
|
166
|
+
|
|
167
|
+
def detect_intent(self, text: str, min_confidence: float = config.MATCH_CONFIDENCE_THRESHOLD) -> IntentResult:
|
|
168
|
+
return self.intent_detector.detect_intent(text, min_confidence=min_confidence)
|
|
169
|
+
|
|
170
|
+
def search_by_emotion(self, emotion: str, limit: int = 20) -> list[dict]:
|
|
171
|
+
return self.search_engine.search_by_emotion(emotion, limit=limit)
|
|
172
|
+
|
|
173
|
+
def search_by_intent(self, intent: str, limit: int = 20) -> list[dict]:
|
|
174
|
+
return self.search_engine.search_by_intent(intent, limit=limit)
|
|
175
|
+
|
|
176
|
+
def search_by_keyword(self, keyword: str, limit: int = 20) -> list[dict]:
|
|
177
|
+
return self.search_engine.search_by_keyword(keyword, limit=limit)
|
|
178
|
+
|
|
179
|
+
def translate_pipeline(
|
|
180
|
+
self,
|
|
181
|
+
text: str,
|
|
182
|
+
min_confidence: float = config.MATCH_CONFIDENCE_THRESHOLD,
|
|
183
|
+
top_n: int = config.PIPELINE_TOP_N,
|
|
184
|
+
session_id: str | None = None,
|
|
185
|
+
explain: bool = False,
|
|
186
|
+
) -> PipelineResult:
|
|
187
|
+
"""
|
|
188
|
+
Run the full Phase 1 structured pipeline: normalize, search, best-
|
|
189
|
+
match selection, intent, emotion, confidence, translation -- in one
|
|
190
|
+
call. Falls back to the top ``top_n`` candidates instead of a single
|
|
191
|
+
answer when confidence is below ``min_confidence``.
|
|
192
|
+
|
|
193
|
+
Pass ``explain=True`` for the Phase 2 developer-facing explanation
|
|
194
|
+
(runner-up, score margin, full ranking) attached under the
|
|
195
|
+
``"explanation"`` key. Default output is unchanged either way.
|
|
196
|
+
"""
|
|
197
|
+
return self.pipeline.run(
|
|
198
|
+
text,
|
|
199
|
+
min_confidence=min_confidence,
|
|
200
|
+
top_n=top_n,
|
|
201
|
+
session_id=session_id,
|
|
202
|
+
explain=explain,
|
|
203
|
+
)
|
|
204
|
+
|
|
205
|
+
|
|
206
|
+
# ---------------------------------------------------------------------------
|
|
207
|
+
# Module-level convenience functions (backed by lazily-initialized singletons
|
|
208
|
+
# defined in each engine module). These are what `import vernacbridge;
|
|
209
|
+
# vernacbridge.translate(...)` calls.
|
|
210
|
+
# ---------------------------------------------------------------------------
|
|
211
|
+
translate = _module_translate
|
|
212
|
+
detect_emotion = _module_detect_emotion
|
|
213
|
+
detect_intent = _module_detect_intent
|
|
214
|
+
search_by_emotion = _module_search_by_emotion
|
|
215
|
+
search_by_intent = _module_search_by_intent
|
|
216
|
+
search_by_keyword = _module_search_by_keyword
|
|
217
|
+
translate_pipeline = _module_translate_pipeline
|
vernacbridge/cleaner.py
ADDED
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Dataset cleaning: exact and near-duplicate removal.
|
|
3
|
+
|
|
4
|
+
Near-duplicate removal uses RapidFuzz similarity scoring, but naive pairwise
|
|
5
|
+
comparison is O(n^2), which does not scale as the corpus grows into the
|
|
6
|
+
tens of thousands of rows the project vision calls for. To keep this
|
|
7
|
+
practical, rows are first bucketed by a cheap normalized prefix
|
|
8
|
+
(:func:`vernacbridge.utils.block_key`); fuzzy comparison only ever happens
|
|
9
|
+
*within* a bucket, since two sentences with completely different openings
|
|
10
|
+
are virtually never near-duplicates in this domain.
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
from collections import defaultdict
|
|
16
|
+
from dataclasses import dataclass
|
|
17
|
+
|
|
18
|
+
import pandas as pd
|
|
19
|
+
|
|
20
|
+
from . import config
|
|
21
|
+
from .utils import block_key, get_logger, normalize_text, similarity_score
|
|
22
|
+
|
|
23
|
+
logger = get_logger(__name__)
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
@dataclass
|
|
27
|
+
class CleaningReport:
|
|
28
|
+
"""Summary of what a cleaning pass removed from the merged dataset."""
|
|
29
|
+
|
|
30
|
+
rows_in: int
|
|
31
|
+
rows_out: int
|
|
32
|
+
exact_duplicates_removed: int = 0
|
|
33
|
+
near_duplicates_removed: int = 0
|
|
34
|
+
|
|
35
|
+
@property
|
|
36
|
+
def total_removed(self) -> int:
|
|
37
|
+
return self.rows_in - self.rows_out
|
|
38
|
+
|
|
39
|
+
def as_dict(self) -> dict:
|
|
40
|
+
return {
|
|
41
|
+
"rows_in": self.rows_in,
|
|
42
|
+
"rows_out": self.rows_out,
|
|
43
|
+
"total_removed": self.total_removed,
|
|
44
|
+
"exact_duplicates_removed": self.exact_duplicates_removed,
|
|
45
|
+
"near_duplicates_removed": self.near_duplicates_removed,
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
class DatasetCleaner:
|
|
50
|
+
"""Removes empty, exactly duplicated, and near-duplicate rows from a dataset."""
|
|
51
|
+
|
|
52
|
+
def __init__(
|
|
53
|
+
self,
|
|
54
|
+
near_duplicate_threshold: float = config.NEAR_DUPLICATE_THRESHOLD,
|
|
55
|
+
match_column: str = "tanglish_input",
|
|
56
|
+
) -> None:
|
|
57
|
+
self.near_duplicate_threshold = near_duplicate_threshold
|
|
58
|
+
self.match_column = match_column
|
|
59
|
+
|
|
60
|
+
def remove_exact_duplicates(self, df: pd.DataFrame) -> tuple[pd.DataFrame, int]:
|
|
61
|
+
"""Drop rows that are byte-for-byte duplicates of an earlier row."""
|
|
62
|
+
normalized = df[self.match_column].astype(str).map(normalize_text)
|
|
63
|
+
dup_mask = normalized.duplicated(keep="first")
|
|
64
|
+
removed = int(dup_mask.sum())
|
|
65
|
+
return df.loc[~dup_mask].copy(), removed
|
|
66
|
+
|
|
67
|
+
def remove_near_duplicates(self, df: pd.DataFrame) -> tuple[pd.DataFrame, int]:
|
|
68
|
+
"""
|
|
69
|
+
Drop rows whose ``match_column`` value is a near-duplicate (RapidFuzz
|
|
70
|
+
similarity >= threshold) of an earlier-seen row in the same bucket.
|
|
71
|
+
The first occurrence of each near-duplicate cluster is kept.
|
|
72
|
+
"""
|
|
73
|
+
buckets: dict[str, list[str]] = defaultdict(list)
|
|
74
|
+
keep_mask: list[bool] = []
|
|
75
|
+
|
|
76
|
+
for value in df[self.match_column].astype(str):
|
|
77
|
+
normalized = normalize_text(value)
|
|
78
|
+
key = block_key(value)
|
|
79
|
+
bucket = buckets[key]
|
|
80
|
+
|
|
81
|
+
is_near_duplicate = any(
|
|
82
|
+
similarity_score(normalized, seen) >= self.near_duplicate_threshold
|
|
83
|
+
for seen in bucket
|
|
84
|
+
)
|
|
85
|
+
|
|
86
|
+
keep_mask.append(not is_near_duplicate)
|
|
87
|
+
if not is_near_duplicate:
|
|
88
|
+
bucket.append(normalized)
|
|
89
|
+
|
|
90
|
+
removed = keep_mask.count(False)
|
|
91
|
+
return df.loc[keep_mask].copy(), removed
|
|
92
|
+
|
|
93
|
+
def clean(self, df: pd.DataFrame) -> tuple[pd.DataFrame, CleaningReport]:
|
|
94
|
+
"""Run the full cleaning pipeline: exact dedup, then near-dedup."""
|
|
95
|
+
rows_in = len(df)
|
|
96
|
+
|
|
97
|
+
df, exact_removed = self.remove_exact_duplicates(df)
|
|
98
|
+
df, near_removed = self.remove_near_duplicates(df)
|
|
99
|
+
|
|
100
|
+
report = CleaningReport(
|
|
101
|
+
rows_in=rows_in,
|
|
102
|
+
rows_out=len(df),
|
|
103
|
+
exact_duplicates_removed=exact_removed,
|
|
104
|
+
near_duplicates_removed=near_removed,
|
|
105
|
+
)
|
|
106
|
+
logger.info("Cleaning complete: %s", report.as_dict())
|
|
107
|
+
return df.reset_index(drop=True), report
|
vernacbridge/config.py
ADDED
|
@@ -0,0 +1,154 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Central configuration for VernacBridge.
|
|
3
|
+
|
|
4
|
+
Every tunable constant lives here so behaviour can be adjusted (or overridden
|
|
5
|
+
by downstream applications) without hunting through the codebase. Paths are
|
|
6
|
+
resolved relative to the installed package by default but can be overridden
|
|
7
|
+
per-instance by the classes that consume them (see ``dataset_engine.py`` and
|
|
8
|
+
``core.py``).
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
from pathlib import Path
|
|
14
|
+
|
|
15
|
+
# --------------------------------------------------------------------------
|
|
16
|
+
# Filesystem layout
|
|
17
|
+
# --------------------------------------------------------------------------
|
|
18
|
+
PACKAGE_ROOT: Path = Path(__file__).resolve().parent
|
|
19
|
+
|
|
20
|
+
#: Directory that is scanned for raw, per-domain CSV datasets. Lives inside
|
|
21
|
+
#: the package itself (not the project root) so it's included as real
|
|
22
|
+
#: package data and discoverable the same way whether running from a
|
|
23
|
+
#: source checkout, an editable install, or a normal (non-editable) pip
|
|
24
|
+
#: install -- all three cases resolve relative to this package's own
|
|
25
|
+
#: location, which is always correct regardless of where pip put it.
|
|
26
|
+
DEFAULT_DATASET_DIR: Path = PACKAGE_ROOT / "datasets"
|
|
27
|
+
|
|
28
|
+
#: Location of the single, cleaned dataset produced by the DatasetEngine.
|
|
29
|
+
DEFAULT_MASTER_DATASET_PATH: Path = DEFAULT_DATASET_DIR / "master_dataset.csv"
|
|
30
|
+
|
|
31
|
+
#: Location of the JSON report written after every dataset build.
|
|
32
|
+
DEFAULT_BUILD_REPORT_PATH: Path = DEFAULT_DATASET_DIR / "build_report.json"
|
|
33
|
+
|
|
34
|
+
# --------------------------------------------------------------------------
|
|
35
|
+
# Dataset schema
|
|
36
|
+
# --------------------------------------------------------------------------
|
|
37
|
+
#: Columns every raw dataset CSV must contain.
|
|
38
|
+
REQUIRED_COLUMNS: tuple[str, ...] = (
|
|
39
|
+
"tanglish_input",
|
|
40
|
+
"english_output",
|
|
41
|
+
"intent",
|
|
42
|
+
"emotion",
|
|
43
|
+
"keywords",
|
|
44
|
+
"urgency",
|
|
45
|
+
"confidence",
|
|
46
|
+
)
|
|
47
|
+
|
|
48
|
+
#: Columns that must not be blank/NaN on any retained row.
|
|
49
|
+
NON_NULLABLE_COLUMNS: tuple[str, ...] = (
|
|
50
|
+
"tanglish_input",
|
|
51
|
+
"english_output",
|
|
52
|
+
"intent",
|
|
53
|
+
"emotion",
|
|
54
|
+
)
|
|
55
|
+
|
|
56
|
+
# --------------------------------------------------------------------------
|
|
57
|
+
# Label vocabularies
|
|
58
|
+
# --------------------------------------------------------------------------
|
|
59
|
+
# These vocabularies are intentionally kept in one place so that adding a
|
|
60
|
+
# new domain (e.g. "health", "finance") only requires appending an intent
|
|
61
|
+
# here plus dropping a new CSV into the datasets folder -- no other code
|
|
62
|
+
# needs to change.
|
|
63
|
+
VALID_EMOTIONS: frozenset[str] = frozenset(
|
|
64
|
+
{
|
|
65
|
+
"angry",
|
|
66
|
+
"anxious",
|
|
67
|
+
"calm",
|
|
68
|
+
"confused",
|
|
69
|
+
"excited",
|
|
70
|
+
"fear",
|
|
71
|
+
"guilty",
|
|
72
|
+
"happy",
|
|
73
|
+
"hopeful",
|
|
74
|
+
"lonely",
|
|
75
|
+
"sad",
|
|
76
|
+
"stressed",
|
|
77
|
+
}
|
|
78
|
+
)
|
|
79
|
+
|
|
80
|
+
VALID_INTENTS: frozenset[str] = frozenset(
|
|
81
|
+
{
|
|
82
|
+
"career",
|
|
83
|
+
"daily_conversation",
|
|
84
|
+
"education",
|
|
85
|
+
"emergency",
|
|
86
|
+
"family",
|
|
87
|
+
"mental_health",
|
|
88
|
+
"motivation",
|
|
89
|
+
"relationship",
|
|
90
|
+
}
|
|
91
|
+
)
|
|
92
|
+
|
|
93
|
+
VALID_URGENCY: frozenset[str] = frozenset({"low", "medium", "high"})
|
|
94
|
+
|
|
95
|
+
VALID_CONFIDENCE: frozenset[str] = frozenset({"low", "medium", "high"})
|
|
96
|
+
|
|
97
|
+
# --------------------------------------------------------------------------
|
|
98
|
+
# Cleaning / matching thresholds
|
|
99
|
+
# --------------------------------------------------------------------------
|
|
100
|
+
#: RapidFuzz similarity (0-100) above which two rows are considered
|
|
101
|
+
#: near-duplicates during dataset cleaning.
|
|
102
|
+
NEAR_DUPLICATE_THRESHOLD: float = 92.0
|
|
103
|
+
|
|
104
|
+
#: RapidFuzz similarity (0-100) below which a translate/detect call is
|
|
105
|
+
#: considered "no confident match" and a fallback response is returned.
|
|
106
|
+
MATCH_CONFIDENCE_THRESHOLD: float = 60.0
|
|
107
|
+
|
|
108
|
+
#: Number of closest candidates returned by TranslationPipeline when the
|
|
109
|
+
#: best match falls below MATCH_CONFIDENCE_THRESHOLD.
|
|
110
|
+
PIPELINE_TOP_N: int = 3
|
|
111
|
+
|
|
112
|
+
#: Score gap (0-100) between the winning candidate and its runner-up below
|
|
113
|
+
#: which DatasetIntelligence flags a match as a "close call" -- used only
|
|
114
|
+
#: by explain-mode output (Phase 2); has no effect on matching itself.
|
|
115
|
+
CLOSE_CALL_MARGIN: float = 10.0
|
|
116
|
+
|
|
117
|
+
#: Maximum number of turns MemoryEngine's default InMemoryStore retains per
|
|
118
|
+
#: session before evicting the oldest (Phase 3). Only bounds short-term
|
|
119
|
+
#: (in-RAM) history; a JSONFileStore, if configured, retains everything.
|
|
120
|
+
MEMORY_SHORT_TERM_CAPACITY: int = 50
|
|
121
|
+
|
|
122
|
+
#: Word count at or below which ContextEngine treats input as a possible
|
|
123
|
+
#: follow-up needing prior-turn context (e.g. "Why?") rather than a
|
|
124
|
+
#: complete, standalone sentence (Phase 4). A structural signal, not a
|
|
125
|
+
#: language-specific one -- deliberately not a hardcoded question-word
|
|
126
|
+
#: list, to avoid guessing at Tanglish vocabulary.
|
|
127
|
+
CONTEXT_FOLLOWUP_MAX_WORDS: int = 3
|
|
128
|
+
|
|
129
|
+
#: Number of most-recent prior turns ContextEngine considers when
|
|
130
|
+
#: resolving a follow-up (Phase 4). Starts at 1 (immediately preceding
|
|
131
|
+
#: turn only) -- the simplest case the original spec's example needs.
|
|
132
|
+
CONTEXT_RECENCY_WINDOW: int = 1
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
#: Maximum number of most-recent turns MemoryEngine's default short-term
|
|
136
|
+
#: store retains per session before evicting the oldest (Phase 3). Has no
|
|
137
|
+
#: effect on long-term (persisted) storage, which is uncapped.
|
|
138
|
+
MEMORY_SHORT_TERM_CAPACITY: int = 50
|
|
139
|
+
|
|
140
|
+
#: Number of leading normalized characters used to "bucket" rows before
|
|
141
|
+
#: near-duplicate comparison, keeping cleaning close to O(n) instead of
|
|
142
|
+
#: O(n^2) on large corpora.
|
|
143
|
+
DEDUP_BLOCK_KEY_LENGTH: int = 4
|
|
144
|
+
|
|
145
|
+
# --------------------------------------------------------------------------
|
|
146
|
+
# Encodings attempted (in order) when reading a CSV file.
|
|
147
|
+
# --------------------------------------------------------------------------
|
|
148
|
+
CANDIDATE_ENCODINGS: tuple[str, ...] = ("utf-8", "utf-8-sig", "latin-1")
|
|
149
|
+
|
|
150
|
+
# --------------------------------------------------------------------------
|
|
151
|
+
# Logging
|
|
152
|
+
# --------------------------------------------------------------------------
|
|
153
|
+
LOG_FORMAT: str = "%(asctime)s | %(levelname)-8s | %(name)s | %(message)s"
|
|
154
|
+
LOG_DATE_FORMAT: str = "%Y-%m-%d %H:%M:%S"
|
vernacbridge/context.py
ADDED
|
@@ -0,0 +1,132 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Context Engine (Phase 4).
|
|
3
|
+
|
|
4
|
+
Implements :class:`vernacbridge.protocols.ContextEngineProtocol`, the
|
|
5
|
+
interface :class:`~vernacbridge.pipeline.TranslationPipeline` has called
|
|
6
|
+
into since Phase 1 (``self.context_engine.resolve(raw_text, session_id) or
|
|
7
|
+
raw_text``, before matching, with failures propagating) without any
|
|
8
|
+
concrete implementation existing until now.
|
|
9
|
+
|
|
10
|
+
Scope, deliberately limited
|
|
11
|
+
---------------------------
|
|
12
|
+
This is corpus-matching architecture, not a language model -- there is no
|
|
13
|
+
generative or semantic understanding underneath it. A "real" context
|
|
14
|
+
engine could mean deep coreference resolution or topic tracking; building
|
|
15
|
+
that here would be inventing linguistic capability for Tanglish specifically
|
|
16
|
+
that hasn't been validated, and is out of V1's scope regardless.
|
|
17
|
+
|
|
18
|
+
What ``ContextEngine.resolve()`` actually does is mechanical: if the
|
|
19
|
+
current input is short (a *structural* signal -- word count under
|
|
20
|
+
``config.CONTEXT_FOLLOWUP_MAX_WORDS`` -- not a guess at specific Tanglish
|
|
21
|
+
question-words, which would risk encoding vocabulary assumptions this
|
|
22
|
+
project hasn't validated) and there's a confident prior turn on record for
|
|
23
|
+
this session, the prior turn's matched Tanglish sentence is prepended to
|
|
24
|
+
the new text before it reaches MatchEngine. That's the entire mechanism:
|
|
25
|
+
give the existing fuzzy matcher more to work with, using only what was
|
|
26
|
+
already confidently matched before. No new capability is invented beyond
|
|
27
|
+
string composition over an existing memory read.
|
|
28
|
+
|
|
29
|
+
Design decision: ``memory_engine`` is required, not optional
|
|
30
|
+
--------------------------------------------------------------
|
|
31
|
+
Every other TranslationPipeline collaborator (``plugin_manager``,
|
|
32
|
+
``conversation_engine``, and ``memory_engine`` itself) defaults to
|
|
33
|
+
``None`` and is skipped when absent. ``ContextEngine`` does not follow
|
|
34
|
+
that pattern on purpose.
|
|
35
|
+
|
|
36
|
+
The first bug ever found in this project was ``self.engine = engine or
|
|
37
|
+
MatchEngine()`` -- falsy-engine-triggered silent creation of a
|
|
38
|
+
disconnected instance instead of sharing the intended one. ``ContextEngine``
|
|
39
|
+
is at direct risk of the same failure mode: its entire purpose is reading
|
|
40
|
+
memory that ``TranslationPipeline`` writes to elsewhere. If it were allowed
|
|
41
|
+
to default-construct its own private ``MemoryEngine()`` when none was
|
|
42
|
+
given, it would silently see empty history forever, and the failure would
|
|
43
|
+
look exactly like "context resolution just doesn't work" with no obvious
|
|
44
|
+
cause. Requiring ``memory_engine`` at construction forces the caller to be
|
|
45
|
+
explicit: the *same* ``MemoryEngine`` instance must be passed to both
|
|
46
|
+
``TranslationPipeline(memory_engine=m, context_engine=ContextEngine(memory_engine=m))``
|
|
47
|
+
(or the equivalent ``VernacBridge`` kwargs). See
|
|
48
|
+
``tests/test_context.py::TestSharedMemoryRequirement`` for a test that
|
|
49
|
+
specifically exercises the "two different instances" mistake.
|
|
50
|
+
"""
|
|
51
|
+
|
|
52
|
+
from __future__ import annotations
|
|
53
|
+
|
|
54
|
+
from typing import TYPE_CHECKING
|
|
55
|
+
|
|
56
|
+
from . import config
|
|
57
|
+
from .protocols import MemoryEngineProtocol
|
|
58
|
+
from .utils import get_logger
|
|
59
|
+
|
|
60
|
+
if TYPE_CHECKING:
|
|
61
|
+
from .pipeline import PipelineResult
|
|
62
|
+
|
|
63
|
+
logger = get_logger(__name__)
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
class ContextEngine:
|
|
67
|
+
"""
|
|
68
|
+
Default implementation of :class:`vernacbridge.protocols.
|
|
69
|
+
ContextEngineProtocol`.
|
|
70
|
+
|
|
71
|
+
``memory_engine`` is required -- see the module docstring for why this
|
|
72
|
+
diverges from every other injectable collaborator in the pipeline.
|
|
73
|
+
"""
|
|
74
|
+
|
|
75
|
+
def __init__(
|
|
76
|
+
self,
|
|
77
|
+
memory_engine: MemoryEngineProtocol,
|
|
78
|
+
max_followup_words: int = config.CONTEXT_FOLLOWUP_MAX_WORDS,
|
|
79
|
+
recency_window: int = config.CONTEXT_RECENCY_WINDOW,
|
|
80
|
+
) -> None:
|
|
81
|
+
if memory_engine is None:
|
|
82
|
+
raise ValueError(
|
|
83
|
+
"ContextEngine requires a memory_engine -- pass the SAME "
|
|
84
|
+
"MemoryEngine instance given to TranslationPipeline/"
|
|
85
|
+
"VernacBridge, so context resolution reads the history "
|
|
86
|
+
"that's actually being recorded. See the module docstring "
|
|
87
|
+
"for why this argument has no default."
|
|
88
|
+
)
|
|
89
|
+
self.memory_engine = memory_engine
|
|
90
|
+
self.max_followup_words = max_followup_words
|
|
91
|
+
self.recency_window = recency_window
|
|
92
|
+
|
|
93
|
+
def resolve(self, text: str, session_id: str | None = None) -> str:
|
|
94
|
+
"""
|
|
95
|
+
Return ``text`` unchanged unless it looks like a short follow-up
|
|
96
|
+
AND a usable (confident, previously-matched) prior turn exists for
|
|
97
|
+
``session_id`` -- in which case, that prior turn's matched
|
|
98
|
+
Tanglish sentence is prepended.
|
|
99
|
+
"""
|
|
100
|
+
stripped = (text or "").strip()
|
|
101
|
+
|
|
102
|
+
if not stripped:
|
|
103
|
+
return text
|
|
104
|
+
|
|
105
|
+
if len(stripped.split()) > self.max_followup_words:
|
|
106
|
+
return text # looks like a complete sentence -- leave it alone
|
|
107
|
+
|
|
108
|
+
if session_id is None:
|
|
109
|
+
return text # nothing to correlate history against
|
|
110
|
+
|
|
111
|
+
history: list["PipelineResult"] = self.memory_engine.recall(session_id)
|
|
112
|
+
if not history:
|
|
113
|
+
return text
|
|
114
|
+
|
|
115
|
+
recent = history[-self.recency_window:] if self.recency_window > 0 else []
|
|
116
|
+
usable_sentences = [
|
|
117
|
+
turn["matched_sentence"]
|
|
118
|
+
for turn in recent
|
|
119
|
+
if turn.get("is_confident") and turn.get("matched_sentence")
|
|
120
|
+
]
|
|
121
|
+
|
|
122
|
+
if not usable_sentences:
|
|
123
|
+
return text # nothing confident enough to fuse in
|
|
124
|
+
|
|
125
|
+
resolved = " ".join([*usable_sentences, stripped])
|
|
126
|
+
logger.info(
|
|
127
|
+
"Resolved follow-up for session_id=%r: %r -> %r",
|
|
128
|
+
session_id,
|
|
129
|
+
text,
|
|
130
|
+
resolved,
|
|
131
|
+
)
|
|
132
|
+
return resolved
|