bhashabid 1.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- bhashabid/__init__.py +44 -0
- bhashabid-1.0.0.dist-info/METADATA +279 -0
- bhashabid-1.0.0.dist-info/RECORD +69 -0
- bhashabid-1.0.0.dist-info/WHEEL +5 -0
- bhashabid-1.0.0.dist-info/entry_points.txt +3 -0
- bhashabid-1.0.0.dist-info/licenses/LICENSE +21 -0
- bhashabid-1.0.0.dist-info/licenses/LICENSE.txt +21 -0
- bhashabid-1.0.0.dist-info/top_level.txt +2 -0
- tools/__init__.py +1 -0
- tools/adversarial_generator.py +122 -0
- tools/assets/adapters/claude/claude_system_prompt.txt +9 -0
- tools/assets/adapters/cursor/bangla-expert.mdc +37 -0
- tools/assets/lexicon/bn_vocab_78k.txt.gz +0 -0
- tools/assets/skills/bangla-expert/SKILL.md +38 -0
- tools/assets/skills/bangla-expert/STANDALONE_ALL_IN_ONE.md +95 -0
- tools/assets/skills/bangla-expert/references/academic_legal_medical.md +46 -0
- tools/assets/skills/bangla-expert/references/constitutional_writs.md +40 -0
- tools/assets/skills/bangla-expert/references/creative_writing.md +48 -0
- tools/assets/skills/bangla-expert/references/ecommerce_fintech_civic.md +26 -0
- tools/assets/skills/bangla-expert/references/grammar_registers.md +17 -0
- tools/assets/skills/bangla-expert/references/journalism_domains.md +21 -0
- tools/assets/skills/bangla-expert/references/numerals.md +26 -0
- tools/assets/skills/bangla-expert/references/official_letters.md +32 -0
- tools/assets/skills/bangla-expert/references/orthography.md +15 -0
- tools/audit_duplicates.py +88 -0
- tools/autonomous_scraper.py +566 -0
- tools/bn_legal/__init__.py +10 -0
- tools/bn_legal/writs.py +118 -0
- tools/bn_lint/__init__.py +5 -0
- tools/bn_lint/calques.py +103 -0
- tools/bn_lint/guru_chandali.py +95 -0
- tools/bn_lint/honorifics.py +257 -0
- tools/bn_lint/linter.py +24 -0
- tools/bn_lint/models.py +17 -0
- tools/bn_normalize/__init__.py +20 -0
- tools/bn_normalize/lexicon.py +278 -0
- tools/bn_normalize/normalizer.py +54 -0
- tools/bn_normalize/orthography.py +169 -0
- tools/bn_normalize/profiles.py +43 -0
- tools/bn_normalize/punctuation.py +77 -0
- tools/bn_normalize/repair.py +249 -0
- tools/bn_normalize/unicode.py +64 -0
- tools/bn_numbers/__init__.py +22 -0
- tools/bn_numbers/currency.py +46 -0
- tools/bn_numbers/dates.py +29 -0
- tools/bn_numbers/decimals.py +39 -0
- tools/bn_numbers/digits.py +59 -0
- tools/bn_numbers/fractions.py +65 -0
- tools/bn_numbers/ordinals.py +24 -0
- tools/bn_numbers/words.py +100 -0
- tools/bn_prosody/__init__.py +15 -0
- tools/bn_prosody/metre.py +137 -0
- tools/bn_prosody/syllable.py +125 -0
- tools/card_synthesizer.py +475 -0
- tools/cli.py +574 -0
- tools/corpus_harvester.py +282 -0
- tools/daily_practice_runner.py +558 -0
- tools/generate_infographic_poster.py +271 -0
- tools/human_reviewer.py +180 -0
- tools/learning_benchmark_ledger.py +239 -0
- tools/learning_curve_validator.py +166 -0
- tools/mcp_server/__init__.py +4 -0
- tools/mcp_server/__main__.py +6 -0
- tools/mcp_server/server.py +316 -0
- tools/mistake_synthesizer.py +331 -0
- tools/review_packet_generator.py +114 -0
- tools/skill_distiller.py +131 -0
- tools/skill_installer.py +566 -0
- tools/verify_provenance.py +72 -0
bhashabid/__init__.py
ADDED
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Bhashabid (ভাষাবিদ) — Deterministic Bengali Linguistic Coprocessor Engine
|
|
3
|
+
Pure-Python, Zero-LLM, High-Throughput Bengali Natural Language Processor.
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
from tools.bn_normalize.repair import repair_text as repair
|
|
7
|
+
from tools.bn_normalize.normalizer import normalize, normalize_text
|
|
8
|
+
from tools.bn_normalize.unicode import canonicalize_unicode, is_nfc_normalized
|
|
9
|
+
from tools.bn_lint.linter import lint_text as lint
|
|
10
|
+
from tools.bn_normalize.lexicon import get_default_lexicon, bengali_edit_distance
|
|
11
|
+
from tools.bn_numbers import format_lakh_crore as format_number, number_to_bangla_words, format_currency_words
|
|
12
|
+
from tools.bn_prosody import scan_line, verify_payar
|
|
13
|
+
|
|
14
|
+
__version__ = "1.0.0"
|
|
15
|
+
|
|
16
|
+
def spellcheck(text_or_word: str):
|
|
17
|
+
"""
|
|
18
|
+
Offline phonetic spellchecker.
|
|
19
|
+
If a single word is provided, returns list of top correction suggestions.
|
|
20
|
+
If a sentence/text is provided, returns list of flagged issue dictionaries.
|
|
21
|
+
"""
|
|
22
|
+
lex = get_default_lexicon()
|
|
23
|
+
text_or_word = text_or_word.strip()
|
|
24
|
+
if " " in text_or_word:
|
|
25
|
+
return lex.audit_text(text_or_word)
|
|
26
|
+
return lex.suggest_corrections(text_or_word)
|
|
27
|
+
|
|
28
|
+
__all__ = [
|
|
29
|
+
"repair",
|
|
30
|
+
"normalize",
|
|
31
|
+
"normalize_text",
|
|
32
|
+
"canonicalize_unicode",
|
|
33
|
+
"is_nfc_normalized",
|
|
34
|
+
"lint",
|
|
35
|
+
"spellcheck",
|
|
36
|
+
"format_number",
|
|
37
|
+
"number_to_bangla_words",
|
|
38
|
+
"format_currency_words",
|
|
39
|
+
"scan_line",
|
|
40
|
+
"verify_payar",
|
|
41
|
+
"get_default_lexicon",
|
|
42
|
+
"bengali_edit_distance",
|
|
43
|
+
"__version__"
|
|
44
|
+
]
|
|
@@ -0,0 +1,279 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: bhashabid
|
|
3
|
+
Version: 1.0.0
|
|
4
|
+
Summary: The Deterministic Bengali linguistic coprocessor engine & progressive skill layer with a Supervised Self-Learning Distillation Loop.
|
|
5
|
+
Author-email: Ashikur Rahman Bhuiyan <ashikur.bhuiyan@brainstation-23.com>, Bhashabid Contributors <contributors@bhashabid.org>
|
|
6
|
+
License: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/ashik1291/bhashabid
|
|
8
|
+
Project-URL: Bug Tracker, https://github.com/ashik1291/bhashabid/issues
|
|
9
|
+
Classifier: Programming Language :: Python :: 3
|
|
10
|
+
Classifier: Operating System :: OS Independent
|
|
11
|
+
Classifier: Natural Language :: Bengali
|
|
12
|
+
Classifier: Topic :: Text Processing :: Linguistic
|
|
13
|
+
Requires-Python: >=3.10
|
|
14
|
+
Description-Content-Type: text/markdown
|
|
15
|
+
License-File: LICENSE
|
|
16
|
+
License-File: LICENSE.txt
|
|
17
|
+
Requires-Dist: pyyaml>=6.0
|
|
18
|
+
Dynamic: license-file
|
|
19
|
+
|
|
20
|
+
# Bhashabid (ভাষাবিদ) — The Deterministic Bengali Language Coprocessor Engine
|
|
21
|
+
|
|
22
|
+
### *Deterministic Bengali linguistic coprocessor engine & progressive skill layer with a Supervised Self-Learning Distillation Loop.*
|
|
23
|
+
|
|
24
|
+
[](LICENSE)
|
|
25
|
+
[](https://python.org)
|
|
26
|
+
[](https://github.com/ashik1291/bhashabid)
|
|
27
|
+
[](eval/unit)
|
|
28
|
+
[](knowledge/cards)
|
|
29
|
+
[](skills/bangla-expert/SKILL.md)
|
|
30
|
+
|
|
31
|
+
> **Bhashabid** (ভাষাবিদ) is the **first-ever open-source Bengali language coprocessor engine** featuring a **constitutionally supervised self-learning loop**. Operating as an external cognitive and deterministic coprocessor without retraining model weights, it continuously harvests real-world anomalies, synthesizes candidate rules, and distills high-level invariants under human verification (Principle P2) and a strict < 390 token ceiling—systematically elevating the accuracy, orthography, and native fluency of AI-generated Bengali across **17 specialized professional sectors**.
|
|
32
|
+
|
|
33
|
+
---
|
|
34
|
+
|
|
35
|
+
## 🎯 What Bhashabid Does & What Problem It Solves Uniquely
|
|
36
|
+
|
|
37
|
+
### The Fundamental Problem with Frontier LLMs in Bengali
|
|
38
|
+
Frontier large language models (GPT-4o, Claude 3.5 Sonnet, Gemini 1.5 Pro, DeepSeek V3) possess vast latent Bengali vocabularies, but their generative decoders suffer from four structural vulnerabilities:
|
|
39
|
+
1. **English Syntactic Calquing**: Literal translation of English passive voice using *"দ্বারা"* (*"খাদ্যমূল্য বৃদ্ধির দ্বারা চালিত"*), clumsy modal verbs, and unidiomatic idioms (*"ঝোপের চারপাশে পেটানো"* for *beat around the bush*).
|
|
40
|
+
2. **Honorific & Register Discordance**: Inconsistent pronominal and verbal honorific agreement (mixing informal *তারা* with honorific *দিয়েছিলেন*, or *তুমি* with *নেন*), and breaking *Sadhu-Cholito* harmony (*Guru-chandali* दोष).
|
|
41
|
+
3. **Probabilistic Failure on Exact Rules**: LLMs frequently predict incorrect compound numerals (*"নব্বই নয়"* instead of *"নিরানব্বই"*), format currency with Western 3-digit commas instead of South Asian lakh/crore commas (`1,234,567` instead of `১২,৩৪,৫৬৭`), and fail to terminate sentences with Dari (`।`).
|
|
42
|
+
4. **Context Window Dilution**: Conventional approaches dump 10,000+ token monolithic prompt guides into the context window, causing catastrophic instruction forgetting and ballooning API costs.
|
|
43
|
+
|
|
44
|
+
### How Bhashabid Solves This Uniquely
|
|
45
|
+
Bhashabid does not attempt to train a new model from scratch. Instead, it pairs with frontier models as a **high-speed, deterministic coprocessor and progressive skill layer**:
|
|
46
|
+
- **Deterministic Offloading via Python & MCP**: Exact rules (0–99 irregular numbers, lakh/crore currency words, Unicode NFC normalization, Dari punctuation, 14-akshar Payar metric scansion) run in local Python algorithms at **500,000 words/second with 100% mathematical precision and $0.00 token cost**.
|
|
47
|
+
- **Progressive Disclosure (< 390 Token Budget)**: The core skill kernel is strictly programmatically capped at **< 390 tokens** (currently **298 true BPE tokens** on `cl100k_base`). Deep domain knowledge is segmented into **9 isolated reference guides** loaded strictly on demand.
|
|
48
|
+
- **Closed-Loop Learning Flywheel (Principle P2)**: When real-world scraping or practice runs encounter linguistic anomalies, candidate cards are synthesized automatically. Before entering production, every single card must undergo **cryptographic human native-speaker review (P2)** with SHA-256 signature verification.
|
|
49
|
+
|
|
50
|
+
---
|
|
51
|
+
|
|
52
|
+
## 📊 Side-by-Side Comparison: Frontier LLMs vs. Bhashabid
|
|
53
|
+
|
|
54
|
+
| Linguistic Dimension | Where Frontier Models Fail (Raw LLM) | What Bhashabid Enforces (Coprocessor) | Concrete Real-World Example |
|
|
55
|
+
| :--- | :--- | :--- | :--- |
|
|
56
|
+
| **1. Irregular Cardinal Numbers** | Compounds numbers literally or uses decimal digit translation (*নব্বই নয়*). | Deterministic lookup for all 0–99 irregular cardinals, fractions (*দেড়, আড়াই*). | Input `99` ➔ Raw: *"নব্বই নয়"* ❌ <br>Bhashabid: **"নিরানব্বই"** ✔️ |
|
|
57
|
+
| **2. South Asian Currency Grouping** | Western 3-digit comma grouping (`1,234,567.50`). | Exact South Asian lakh/crore grouping and currency in words. | Input `1234567.50` ➔ Raw: *"1,234,567.50"* ❌ <br>Bhashabid: **"১২,৩৪,৫৬৭.৫০ (বারো লাখ চৌত্রিশ হাজার পাঁচশত সাতষট্টি টাকা পঞ্চাশ পয়সা)"** ✔️ |
|
|
58
|
+
| **3. English Passive Voice Calques** | Translates passives with *"দ্বারা"* (*"দ্বারা চালিত"*, *"দ্বারা অনুষ্ঠিত"*). | Real-time static linter converts passive calques into natural active causal phrasing. | Raw: *"খাদ্যমূল্য বৃদ্ধির দ্বারা চালিত"* ❌ <br>Bhashabid: **"খাদ্যপণ্যের মূল্যবৃদ্ধির কারণে... দাঁড়িয়েছে"** ✔️ |
|
|
59
|
+
| **4. Pronominal & Verbal Honorifics** | Mismatches subject pronouns and verb honorific levels (*"তারা ফেরত দিয়েছিলেন"*). | Enforces strict grammatical concordance (*তারা ... ফেরত দিয়েছিল* vs *তাঁরা ... ফেরত দিয়েছিলেন*). | Raw: *"তারা অর্থ ফেরত দিয়েছিলেন"* ❌ <br>Bhashabid: **"তাঁরা সেই অর্থ ফেরত দিয়েছিলেন"** ✔️ |
|
|
60
|
+
| **5. Judicial Writ Formulas (Article 102)** | Hallucinates colloquial phrases (*"জিজ্ঞেস করে রুল দিয়েছে"*). | Enforces Supreme Court of Bangladesh canonical Rule Nisi and prayer formulas. | Raw: *"কেন বেআইনি হবে না জিজ্ঞেস করে রুল দিয়েছে"* ❌ <br>Bhashabid: **"কেন বেআইনি ঘোষণা করা হবে না, তা জানতে চেয়ে রুল জারি করেছেন"** ✔️ |
|
|
61
|
+
| **6. Administrative Memos & Petitions** | Calques *"Dear Sir"* as *"প্রিয় মহাশয়"*; uses broken closings. | Enforces *Bangladesh Secretariat Instructions* (*সচিবালয় নির্দেশমালা*). | Raw: *"প্রিয় মহাশয়, আশা করি ভালো আছেন"* ❌ <br>Bhashabid: **"বরাবর... বিষয়:... জনাব, বিনীত নিবেদন এই যে..."** ✔️ |
|
|
62
|
+
| **7. Customer Support & Omnichannel** | Produces hybrid Banglish jargon (*"টিকেট রিজলভ করতে এজেন্ট এসাইন করা হয়েছে"*). | Standardizes professional customer care Bengali while preserving tracking IDs (`TCK-942`). | Raw: *"Query ক্লোজ করা হয়েছে, ফিডব্যাক দিন"* ❌ <br>Bhashabid: **"আপনার অভিযোগ ট্র্যাকিং নম্বর TCK-942 দ্রুত সমাধানের লক্ষ্যে পর্যালোচনা করা হচ্ছে"** ✔️ |
|
|
63
|
+
| **8. MFS & FinTech Protocols** | Translates wallet terms literally or generates unsafe financial instructions. | Formulates Bangladesh Bank PSD & MRA compliant digital wallet disclosures. | Raw: *"ক্যাশ আউট ফি চার্জ করা হবে"* ❌ <br>Bhashabid: **"প্রতি হাজারে নির্ধারিত ক্যাশ-আউট চার্জ প্রযোজ্য হবে"** ✔️ |
|
|
64
|
+
| **9. Bangla Academy Orthography** | Spells foreign loanwords with *"ষ"* (*ষ্টেশন, পোষ্ট*); uses long *"ী"* in non-tatsama words (*পাখী*). | Deterministically normalizes text to modern Bangla Academy standards. | Raw: *"রেল ষ্টেশনে পাখী ডাকছে"* ❌ <br>Bhashabid: **"রেল স্টেশনে পাখি ডাকছে"** ✔️ |
|
|
65
|
+
| **10. Sentence Punctuation** | Terminates sentences with ASCII periods (`.`) instead of Dari (`।`). | Deterministically converts sentence-ending periods to Dari while protecting decimal numbers (`৩.১৪`). | Raw: *"তিনি বাড়ি গেলেন."* ❌ <br>Bhashabid: **"তিনি বাড়ি গেলেন।"** ✔️ |
|
|
66
|
+
| **11. Classical 14-Akshar Payar Scansion** | Emits pseudo-verse with arbitrary line breaks and broken syllable rhythm. | Deterministic phonological syllabifier (*মুক্তদল* vs *রুদ্ধদল*) verifies exact 8+6=14 akshar Payar cadence. | Raw: Broken 17-syllable prose line ❌ <br>Bhashabid: **Validates exact 8+6=14 akshar Payar couplets** ✔️ |
|
|
67
|
+
| **12. Land Registry & Deeds (Saf-Kabala)** | Uses informal phrasing (*"জমির বিক্রির কাগজ"*). | Enforces statutory Registration Act deed taxonomy (সাফ-কবলা, মৌজা, খতিয়ান, দাগ নং, চৌহদ্দি). | Raw: *"জমির বিক্রির কাগজ হস্তান্তর করলাম"* ❌ <br>Bhashabid: **"রেজিস্ট্রিকৃত সাফ-কবলা বিক্রয় দলিলমূলে চূড়ান্ত দখল বুঝিয়ে দেওয়া হলো"** ✔️ |
|
|
68
|
+
| **13. Disaster Warning & Cyclone Signals** | Translates storm surge as *"ঝড় বৃদ্ধি"*; hallucinates warning signals. | Enforces official BMD maritime warning signals (মহাবিপদ সংকেত ১-১০) and canonical *জলোচ্ছ্বাস*. | Raw: *"উপকূলীয় অঞ্চলে ঝড় বৃদ্ধি দেখা যেতে পারে"* ❌ <br>Bhashabid: **"উপকূলীয় জেলাসমূহে ৫-৭ ফুট উচ্চতার জলোচ্ছ্বাসের আশঙ্কা রহিয়াছে"** ✔️ |
|
|
69
|
+
| **14. NCTB Creative Pedagogy** | Generates arbitrary essay prompts without cognitive grading. | Formulates canonical NCTB 4-tier Bloom's taxonomy framework (উদ্দীপক, জ্ঞান, অনুধাবন, প্রয়োগ, উচ্চতর দক্ষতা). | Raw: Unstructured single essay prompts ❌ <br>Bhashabid: **Structured 4-tier rubric (ক/খ/গ/ঘ)** ✔️ |
|
|
70
|
+
|
|
71
|
+
---
|
|
72
|
+
|
|
73
|
+
## 🏛️ The 4 Pioneering Architectural Pillars
|
|
74
|
+
|
|
75
|
+
```mermaid
|
|
76
|
+
graph TD
|
|
77
|
+
UserReq["User Prompt / AI Agent Task"] --> Router["Core Skill Kernel (298 BPE Tokens)"]
|
|
78
|
+
|
|
79
|
+
subgraph Pillar1 ["Pillar 1: Programmatic Token Ceiling"]
|
|
80
|
+
Router -->|"Strict Budget Guard (< 390)"| ActiveContext["298 True BPE Tokens (cl100k)"]
|
|
81
|
+
end
|
|
82
|
+
|
|
83
|
+
subgraph Pillar3 ["Pillar 3: Modular Domain Isolation"]
|
|
84
|
+
Router -.->|"On-Demand Only"| LegalGuide["constitutional_writs.md (0 tokens if unrelated)"]
|
|
85
|
+
Router -.->|"On-Demand Only"| BankingGuide["journalism_domains.md (0 tokens if unrelated)"]
|
|
86
|
+
Router -.->|"On-Demand Only"| FinTechGuide["ecommerce_fintech_civic.md (0 tokens if unrelated)"]
|
|
87
|
+
Router -.->|"On-Demand Only"| CreativeGuide["creative_writing.md (0 tokens if unrelated)"]
|
|
88
|
+
end
|
|
89
|
+
|
|
90
|
+
subgraph Pillar4 ["Pillar 4: Deterministic MCP Offloading"]
|
|
91
|
+
ActiveContext ==>|"Zero Token Cost ($0.00)"| FastPython["Python Engine (500k words/sec)"]
|
|
92
|
+
FastPython --> ExactNum["Exact 0-99 Numbers (নিরানব্বই)"]
|
|
93
|
+
FastPython --> LakhCrore["Lakh/Crore Comma Grouping (১২,৩৪,৫৬৭)"]
|
|
94
|
+
FastPython --> UnicodeNFC["Unicode NFC & Dari (।) Normalizer"]
|
|
95
|
+
FastPython --> ProsodyPayar["14-Akshar Payar Syllabic Verifier"]
|
|
96
|
+
end
|
|
97
|
+
|
|
98
|
+
subgraph Pillar2 ["Pillar 2: Supervised Self-Learning Loop"]
|
|
99
|
+
RealWorldAnomalies["Harvester: Real-World Scraped Corpora"] --> Synthesizer["tools/card_synthesizer.py"]
|
|
100
|
+
Synthesizer --> ReviewP2["Human Native Sign-Off Gate (Principle P2)"]
|
|
101
|
+
ReviewP2 --> Distiller["tools/skill_distiller.py"]
|
|
102
|
+
Distiller -->|"Extracts Invariants: 1 rule covers 50k phrases"| CoreInvariants["Zero-Bloat Invariant Updates"]
|
|
103
|
+
CoreInvariants --> Router
|
|
104
|
+
end
|
|
105
|
+
```
|
|
106
|
+
|
|
107
|
+
### 1. Hard Programmatic Token Ceiling (< 390 Tokens Core Budget)
|
|
108
|
+
- **Problem**: Monolithic linguistic system prompts (5,000–20,000 tokens) cause severe context dilution, high latency, and high API expenses.
|
|
109
|
+
- **Solution**: The root skill kernel (`skills/bangla-expert/SKILL.md` and `.cursor/rules/bangla-expert.mdc`) is hard-capped programmatically under **390 tokens** (currently **298 true BPE tokens** on `cl100k_base`). It contains only universal structural invariants; domain reference files are retrieved via progressive disclosure.
|
|
110
|
+
|
|
111
|
+
### 2. Generalization Over Memorization (Supervised Closed-Loop Learning)
|
|
112
|
+
- **Problem**: Manually appending example sentences to prompts causes prompt bloat. Un-gated autonomous self-learning leads to hallucinated grammar rules.
|
|
113
|
+
- **Solution**: Bhashabid executes a closed loop (`harvest` ➔ `synthesize-cards` ➔ `human review (P2)` ➔ `distill`). A single high-level invariant covers tens of thousands of sentence permutations. Knowledge cards cannot enter production without **cryptographic human sign-off**.
|
|
114
|
+
|
|
115
|
+
### 3. Modular Domain Isolation (9 Progressive Guides)
|
|
116
|
+
- **Problem**: Exposing all specialized vocabularies simultaneously confuses model style and wastes tokens.
|
|
117
|
+
- **Solution**: 17 sectors are organized into **9 isolated reference guides**. An agent writing poetry burns **0.0 prompt tokens** on Supreme Court writ rules or central bank banking regulations.
|
|
118
|
+
|
|
119
|
+
### 4. Deterministic Python Offloading via MCP ($0.00 Token Cost)
|
|
120
|
+
- **Problem**: LLMs struggle with mathematical lookups, comma spacing, and phonological syllable counting.
|
|
121
|
+
- **Solution**: Bhashabid offloads deterministic operations to local Python algorithms via the **Model Context Protocol (MCP)** and CLI at **500,000 words/second**, with **100% mathematical precision and 0 prompt tokens burned**.
|
|
122
|
+
|
|
123
|
+
---
|
|
124
|
+
|
|
125
|
+
## 📚 The 17 Supported Professional Sectors
|
|
126
|
+
|
|
127
|
+
Bhashabid provides comprehensive domain standards across **17 specialized professional sectors**:
|
|
128
|
+
|
|
129
|
+
| # | Sector | Authoritative Standards & Coverage | Reference Guide |
|
|
130
|
+
| :---: | :--- | :--- | :--- |
|
|
131
|
+
| **1** | **Constitutional Law & Writs** | Constitution of Bangladesh (Article 102), Supreme Court precedents, Rule Nisi show-cause orders, writ prayers (*Habeas Corpus, Mandamus, Certiorari*). | [`constitutional_writs.md`](skills/bangla-expert/references/constitutional_writs.md) |
|
|
132
|
+
| **2** | **Macroeconomics & Banking** | Bangladesh Bank Monetary Policy statements, prudential regulations, *নীতি সুদহার*, *মূল্যস্ফীতি*, active causal financial reporting. | [`journalism_domains.md`](skills/bangla-expert/references/journalism_domains.md) |
|
|
133
|
+
| **3** | **Healthcare & Epidemiology** | Bangla Academy Medical Dictionary, DGHS guidelines, precise clinical dosage intervals, non-stigmatizing mental health phrasing. | [`academic_legal_medical.md`](skills/bangla-expert/references/academic_legal_medical.md) |
|
|
134
|
+
| **4** | **Administrative Governance & Memos** | *Bangladesh Secretariat Instructions* (*সচিবালয় নির্দেশমালা*), formal administrative petitions, opening and closing protocols. | [`official_letters.md`](skills/bangla-expert/references/official_letters.md) |
|
|
135
|
+
| **5** | **Creative Fiction & Spoken Dialogue** | Bangla Academy Standard Grammar, living spoken Cholito (*কথ্য রীতি*), action-embedded dialogue tags, sensory imagery. | [`creative_writing.md`](skills/bangla-expert/references/creative_writing.md) |
|
|
136
|
+
| **6** | **Classical Poetry & Prosody** | Rabindranath Tagore & Probodh Chandra Sen prosody canon, Dalabritta, Matrabritta, 14-akshar Payar couplets (8+6). | [`creative_writing.md`](skills/bangla-expert/references/creative_writing.md) |
|
|
137
|
+
| **7** | **Academic Research & Methodology** | Peer-reviewed citation standards, academic transliteration with Latin in parentheses, thesis structure. | [`academic_legal_medical.md`](skills/bangla-expert/references/academic_legal_medical.md) |
|
|
138
|
+
| **8** | **Land Registry & Deeds** | Transfer of Property Act 1882, Registration Act 1908, Sub-Kabala conveyance, Heba, CS/SA/RS/BS Khatian, plot boundaries (*চৌহদ্দি*). | [`academic_legal_medical.md`](skills/bangla-expert/references/academic_legal_medical.md) |
|
|
139
|
+
| **9** | **Agronomy & Disaster Management** | Department of Agricultural Extension (DAE), BMD official cyclone warning signals (1-10 মহাবিপদ সংকেত), storm surge (*জলোচ্ছ্বাস*). | [`academic_legal_medical.md`](skills/bangla-expert/references/academic_legal_medical.md) |
|
|
140
|
+
| **10** | **ICT & Software Engineering** | Bangladesh Computer Council (BCC) computing standards, Unicode NFC canonical normalization, open-source localization. | [`orthography.md`](skills/bangla-expert/references/orthography.md) |
|
|
141
|
+
| **11** | **Education & NCTB Pedagogy** | National Curriculum and Textbook Board (NCTB) 4-tier Creative Questioning taxonomy (উদ্দীপক, জ্ঞান, অনুধাবন, প্রয়োগ, উচ্চতর দক্ষতা). | [`academic_legal_medical.md`](skills/bangla-expert/references/academic_legal_medical.md) |
|
|
142
|
+
| **12** | **Translation & Calque Naturalization** | De-calquing passive voice (*দ্বারা*), naturalizing English idioms (*বাগধারা ও প্রবচন*), semantic tri-layer model (*অভিধা, লক্ষণা, ব্যঞ্জনা*). | [`grammar_registers.md`](skills/bangla-expert/references/grammar_registers.md) |
|
|
143
|
+
| **13** | **Consumer E-Commerce & Retail Logistics** *(NEW)* | Consumers' Rights Protection Act 2009, e-CAB guidelines, escrow settlement, refund honorific concordance (*তারা বনাম তাঁরা*), courier tracking. | [`ecommerce_fintech_civic.md`](skills/bangla-expert/references/ecommerce_fintech_civic.md) |
|
|
144
|
+
| **14** | **Customer Support & Omnichannel Messaging** *(NEW)* | Telecom & omnichannel standards, administrative ticketing resolution nomenclature, preserving alphanumeric tracking IDs (`TCK-942`). | [`ecommerce_fintech_civic.md`](skills/bangla-expert/references/ecommerce_fintech_civic.md) |
|
|
145
|
+
| **15** | **MFS, Microfinance & Grassroots FinTech** *(NEW)* | Bangladesh Bank Payment Systems Dept (PSD) MFS guidelines, Microcredit Regulatory Authority (MRA) rules, fraud prevention warnings. | [`ecommerce_fintech_civic.md`](skills/bangla-expert/references/ecommerce_fintech_civic.md) |
|
|
146
|
+
| **16** | **Citizen Digital Services & Smart Governance** *(NEW)* | Aspire to Innovate (a2i) Citizen Service Standards, Bangladesh National Portal style, National ID (NID) registration rules, Citizen Charters. | [`ecommerce_fintech_civic.md`](skills/bangla-expert/references/ecommerce_fintech_civic.md) |
|
|
147
|
+
| **17** | **Mass Media & Digital Journalism** *(NEW)* | Bangladesh Press Council Code of Ethics & Stylebook, Press Institute of Bangladesh (PIB) standards, neutral reporting attribution. | [`journalism_domains.md`](skills/bangla-expert/references/journalism_domains.md) |
|
|
148
|
+
|
|
149
|
+
---
|
|
150
|
+
|
|
151
|
+
## 🧪 Proven Accuracy & Verification Metrics
|
|
152
|
+
|
|
153
|
+
Bhashabid enforces an anti-self-deception testing philosophy (Principle P8). Every mistake discovered in practice is preserved in a permanent regression ledger and executed on every commit:
|
|
154
|
+
|
|
155
|
+
| Verification Suite | Target | Observed Result | Status |
|
|
156
|
+
| :--- | :---: | :---: | :---: |
|
|
157
|
+
| **Automated Unit & Regression Suite** | 100% Pass | **102 / 102 Tests Passing** |  |
|
|
158
|
+
| **Principle P2 Provenance Integrity** | 100% Signed | **48 / 48 Cards Reviewed** |  |
|
|
159
|
+
| **Core Skill Token Footprint** | < 390 Tokens | **298 True BPE Tokens** (`cl100k_base`) |  |
|
|
160
|
+
| **Invariant Duplicate Collisions** | 0 Collisions | **0 Duplicate Titles / Summaries / Examples** |  |
|
|
161
|
+
| **Irregular Number Precision (0–99)** | 100.0% | **100.0% Deterministic Match** |  |
|
|
162
|
+
| **Sentence-Ending Dari Enforcement** | 100.0% | **100.0% (Decimal Protected)** |  |
|
|
163
|
+
| **Guru-chandali Concordance Trapping** | 100.0% | **100.0% Trapped with Bidirectional Fixes** |  |
|
|
164
|
+
| **Execution Throughput (Local Python)** | > 100k wps | **~500,000 words/second ($0.00 cost)** |  |
|
|
165
|
+
|
|
166
|
+
---
|
|
167
|
+
|
|
168
|
+
## 🚀 How to Use Bhashabid Today
|
|
169
|
+
|
|
170
|
+
### 1. 1-Command Skill Installation (Antigravity, Claude, Cursor)
|
|
171
|
+
Install the progressive skill along with **all 9 specialized reference guides** into your coding environment:
|
|
172
|
+
```bash
|
|
173
|
+
# Install globally to all detected environments (Antigravity, Claude, and Cursor)
|
|
174
|
+
bhashabid install-skill --target all
|
|
175
|
+
|
|
176
|
+
# Or install specifically:
|
|
177
|
+
bhashabid install-skill --target antigravity
|
|
178
|
+
bhashabid install-skill --target claude
|
|
179
|
+
bhashabid install-skill --target cursor
|
|
180
|
+
|
|
181
|
+
# Uninstall cleanly at any time:
|
|
182
|
+
bhashabid uninstall-skill --target all
|
|
183
|
+
```
|
|
184
|
+
|
|
185
|
+
### 2. Standalone All-In-One Prompt (ChatGPT, Claude Web, Gemini Web, DeepSeek)
|
|
186
|
+
If you are using a web browser interface and cannot install files, copy the single self-contained prompt:
|
|
187
|
+
👉 **[`skills/bangla-expert/STANDALONE_ALL_IN_ONE.md`](skills/bangla-expert/STANDALONE_ALL_IN_ONE.md)**
|
|
188
|
+
|
|
189
|
+
Paste it into ChatGPT Custom Instructions, Claude System Prompts, or Gemini to immediately eliminate machine-translation calques.
|
|
190
|
+
|
|
191
|
+
### 3. Model Context Protocol (MCP) Server
|
|
192
|
+
Expose Bhashabid's deterministic tools via stdio MCP for Claude Desktop, Cursor, Windsurf, or Zed:
|
|
193
|
+
```bash
|
|
194
|
+
bhashabid-mcp
|
|
195
|
+
# or: bhashabid mcp
|
|
196
|
+
```
|
|
197
|
+
**Exposed Deterministic Tools**:
|
|
198
|
+
- `convert_number`: Irregular 0–99 cardinal words, currency words, South Asian comma grouping.
|
|
199
|
+
- `normalize_text`: Unicode NFC canonical decomposition, Dari punctuation, Bangla Academy vowel rules.
|
|
200
|
+
- `lint_text`: Static analysis for Guru-chandali, honorific discordance, and 20+ English calques.
|
|
201
|
+
- `prosody_scan`: Classical scansion (Dalabritta, Matrabritta, Aksharbritta) and 14-akshar Payar validation.
|
|
202
|
+
- `generate_writ_prayer`: Drafts Article 102 High Court constitutional writ prayers in statutory Bengali.
|
|
203
|
+
|
|
204
|
+
### 4. Command-Line Interface (CLI)
|
|
205
|
+
Install globally from PyPI:
|
|
206
|
+
```bash
|
|
207
|
+
pip install bhashabid
|
|
208
|
+
```
|
|
209
|
+
Run any linguistic task instantly:
|
|
210
|
+
```bash
|
|
211
|
+
# Convert numbers to authentic Bengali words
|
|
212
|
+
bhashabid number 99 --format words
|
|
213
|
+
# Output: নিরানব্বই
|
|
214
|
+
|
|
215
|
+
# Format currency in South Asian lakh/crore
|
|
216
|
+
bhashabid number 1234567.50 --format currency
|
|
217
|
+
# Output: বারো লাখ চৌত্রিশ হাজার পাঁচশত সাতষট্টি টাকা পঞ্চাশ পয়সা
|
|
218
|
+
|
|
219
|
+
# Lint text for grammatical, honorific, and calque errors
|
|
220
|
+
bhashabid lint "তারা সেই অর্থ ফেরত দিয়েছিলেন এবং বিষয়টি বিদ্যুৎ দ্বারা চালিত."
|
|
221
|
+
|
|
222
|
+
# Verify 14-akshar Payar poetic metre (8+6)
|
|
223
|
+
bhashabid prosody "কাশীদাস কহে সাধু শুনে পুণ্যবান" --verify-payar
|
|
224
|
+
|
|
225
|
+
# Draft High Court constitutional writ prayers
|
|
226
|
+
bhashabid writ --remedy "detention" --subject "জনাব করিম" --authority "স্বরাষ্ট্র সচিব"
|
|
227
|
+
|
|
228
|
+
# List candidate knowledge cards awaiting human sign-off (Principle P2)
|
|
229
|
+
bhashabid review --list
|
|
230
|
+
```
|
|
231
|
+
|
|
232
|
+
### 5. Python SDK
|
|
233
|
+
```python
|
|
234
|
+
from tools.bn_numbers import number_to_bangla_words, format_lakh_crore
|
|
235
|
+
from tools.bn_normalize import normalize
|
|
236
|
+
from tools.bn_lint import lint_text
|
|
237
|
+
from tools.bn_prosody import verify_payar
|
|
238
|
+
|
|
239
|
+
# Deterministic numerals
|
|
240
|
+
print(number_to_bangla_words(99)) # "নিরানব্বই"
|
|
241
|
+
|
|
242
|
+
# Orthography & Punctuation
|
|
243
|
+
print(normalize("ডাক্তার সাহেব ৩.১৪ গ্রাম ওষুধ দিলেন."))
|
|
244
|
+
# "ডাক্তার সাহেব ৩.১৪ গ্রাম ওষুধ দিলেন।"
|
|
245
|
+
|
|
246
|
+
# Linter checks with bidirectional repair suggestions
|
|
247
|
+
issues = lint_text("তুমি সিদ্ধান্ত নেন")
|
|
248
|
+
for issue in issues:
|
|
249
|
+
print(issue.suggestion) # তুমি সিদ্ধান্ত নাও / আপনি সিদ্ধান্ত নেন
|
|
250
|
+
```
|
|
251
|
+
|
|
252
|
+
---
|
|
253
|
+
|
|
254
|
+
## 🛡️ Constitutional Governance (Principle P2 & P8)
|
|
255
|
+
|
|
256
|
+
1. **Principle P2 (Human Provenance Gate)**: No knowledge card enters production without authenticated native-speaker review, non-empty verified exemplars, and cryptographic SHA-256 signatures. **48 / 48 cards (100%)** are authenticated by lead human arbiter `Ashikur Rahman Bhuiyan`.
|
|
257
|
+
2. **Principle P4 (Deterministic Tools for Rules)**: Rules that can be computed with discrete mathematics or grammars are implemented in code, not left to probabilistic prompt guesses.
|
|
258
|
+
3. **Principle P8 (Anti-Self-Deception & Memory)**: Flaws detected across real-world corpora are permanently logged in [`memory/mistakes/ledger.jsonl`](memory/mistakes/ledger.jsonl) and sealed as automated unit tests in [`eval/unit/`](eval/unit/).
|
|
259
|
+
|
|
260
|
+
---
|
|
261
|
+
|
|
262
|
+
## 📖 Wiki & Documentation Hub
|
|
263
|
+
|
|
264
|
+
Comprehensive architectural specifications, domain style guides, and developer workflows:
|
|
265
|
+
- [Wiki Home & Overview](docs/wiki/Home.md)
|
|
266
|
+
- [Getting Started & Installation Guide](docs/wiki/Getting-Started.md)
|
|
267
|
+
- [The 17 Specialized Professional Sectors](docs/wiki/The-17-Professional-Sectors.md)
|
|
268
|
+
- [Model Context Protocol (MCP) Guide](docs/wiki/Model-Context-Protocol-MCP.md)
|
|
269
|
+
- [Poetic Prosody & Payar Scansion Specification](docs/wiki/Poetic-Prosody-&-Payar.md)
|
|
270
|
+
- [Article 102 Constitutional Writs Guide](docs/wiki/Constitutional-Writs-Article-102.md)
|
|
271
|
+
- [Principle P2 Provenance & Human Review Protocol](docs/wiki/Principle-P2-Provenance-&-Review.md)
|
|
272
|
+
- [Centralized Linguistic Review Hub](docs/review_queue/README.md)
|
|
273
|
+
- [Pending Human Reviews Dashboard](docs/review_queue/PENDING_REVIEWS.md)
|
|
274
|
+
|
|
275
|
+
---
|
|
276
|
+
|
|
277
|
+
## 📄 License
|
|
278
|
+
|
|
279
|
+
This project is licensed under the permissive terms of the **[MIT License](LICENSE)**. See the `LICENSE` file for full copyright and permission notices.
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
bhashabid/__init__.py,sha256=4H5Eec0DpI3sx3kz-9vV2GPi0H0JG2Yd8Oed8VRhAU4,1481
|
|
2
|
+
bhashabid-1.0.0.dist-info/licenses/LICENSE,sha256=1CuXjR9DCQnQB8D9ZRIEfF2iw6VGXWM3zt5caRX9m-k,1110
|
|
3
|
+
bhashabid-1.0.0.dist-info/licenses/LICENSE.txt,sha256=1CuXjR9DCQnQB8D9ZRIEfF2iw6VGXWM3zt5caRX9m-k,1110
|
|
4
|
+
tools/__init__.py,sha256=r-eyI7M1T1zmkn4KCzEpktERHyHHEuxS0vzcMnnSljY,26
|
|
5
|
+
tools/adversarial_generator.py,sha256=nPZqLtCURr3CJSy8vKcMvqX8cwVweeXQJh2rVkh5b5g,5175
|
|
6
|
+
tools/audit_duplicates.py,sha256=wWNF3r3jPkteifrCeEbQT5LQd8s3BpxkBwJ6Nrojti4,3092
|
|
7
|
+
tools/autonomous_scraper.py,sha256=dIQo7l5NA-MY2Ejgv1GOT5pudqk3OEYLQPtg1MktqiA,28230
|
|
8
|
+
tools/card_synthesizer.py,sha256=lfzr7qMsgCBNMZKLx8FDADc7pIg1IjxLLmn0R3-PHiI,22658
|
|
9
|
+
tools/cli.py,sha256=paUxOFi1U2sFPu9zgj6gbLYrY5b-WRHKvvAjUIa64Qo,27521
|
|
10
|
+
tools/corpus_harvester.py,sha256=fSWOqDWtEr-aHS4MhoX6vuO-b-RSlNZcUtw9Vu-6LgY,12450
|
|
11
|
+
tools/daily_practice_runner.py,sha256=6lKEoDlmIqqClmPUFU35C0EZIbO6B1EjHLQcqawZ3WU,31431
|
|
12
|
+
tools/generate_infographic_poster.py,sha256=4Si92SnEH2f1-jCpjLO2QxFstJBYXhfcRJdsvSJMf-k,14753
|
|
13
|
+
tools/human_reviewer.py,sha256=DtesF8r9jLxm2I_gTM5Y6J5srLddjlxwu9-YePQ8dt8,8557
|
|
14
|
+
tools/learning_benchmark_ledger.py,sha256=EiyxKC39Y_zsCcFoiNF6Psz03YkaCBeGLY8xrX2LVrM,10017
|
|
15
|
+
tools/learning_curve_validator.py,sha256=oL2NsKziFmf1yrbuAl9Cu-G5upaMM6CztjGpuuDZ0hE,7565
|
|
16
|
+
tools/mistake_synthesizer.py,sha256=To1cc1yW_j-qOi60fIw3Ao78nMGKw5ySsKSfLJk-RCk,25704
|
|
17
|
+
tools/review_packet_generator.py,sha256=te2nrP9aYR8JyO1p3cqilcBl_Gjuk3giqcjnKhqivrI,5532
|
|
18
|
+
tools/skill_distiller.py,sha256=Z6m9cm-EOWljsgFhlEQl5xYcaN2lxH9tvdGeiVg6s3E,5497
|
|
19
|
+
tools/skill_installer.py,sha256=KejNQEVs7BACfgSsGD9n99im56CbU3kP1H-dOaG5tkM,23754
|
|
20
|
+
tools/verify_provenance.py,sha256=Yju7Ttowk-ZE5ECN2gcIS5upf5Tjt-FcnJ_2JnHCd4g,2765
|
|
21
|
+
tools/assets/adapters/claude/claude_system_prompt.txt,sha256=H0ColI-Xll8P9uz58PXH-KIcwvX2eizZce1dvwVqGQ0,873
|
|
22
|
+
tools/assets/adapters/cursor/bangla-expert.mdc,sha256=0tgmeWmMugN4RGSw63D6Xfwr0DIKW6MBhMo_YqGlADU,2163
|
|
23
|
+
tools/assets/lexicon/bn_vocab_78k.txt.gz,sha256=tuv_p_-8siVzyaODi42QbmTuxYVO2Hkg3ss6OxC6Vug,314225
|
|
24
|
+
tools/assets/skills/bangla-expert/SKILL.md,sha256=0YmZem2vZvXYDVbZJzC9VgT8pTeZfXqRG_BuwYSwkPA,2282
|
|
25
|
+
tools/assets/skills/bangla-expert/STANDALONE_ALL_IN_ONE.md,sha256=6x2h14cmJ5xiX5MxutSdO-Y3V_XZsOFBfNYDN7Xo0Jk,9596
|
|
26
|
+
tools/assets/skills/bangla-expert/references/academic_legal_medical.md,sha256=gLw8tgRcW4hu0O4lxc6VbybNDsldJcczFCe-kCuc3FA,5965
|
|
27
|
+
tools/assets/skills/bangla-expert/references/constitutional_writs.md,sha256=uQUXHhbXeKXe_HYuZYPij-i0RHTXqYuwdsWyRyqCsRE,6247
|
|
28
|
+
tools/assets/skills/bangla-expert/references/creative_writing.md,sha256=8sniM2asp1FAXg0S1uBJRbVUPuqDALaUpluRS1A-6DE,5103
|
|
29
|
+
tools/assets/skills/bangla-expert/references/ecommerce_fintech_civic.md,sha256=Q98r5bzJO7xWymZrlnQh6PjvzNro-S_z4pDO1XTAJxE,4203
|
|
30
|
+
tools/assets/skills/bangla-expert/references/grammar_registers.md,sha256=kcOAIyt-yf92FdglEe_Ove0udCNBwAKJXqyORByaIcE,1121
|
|
31
|
+
tools/assets/skills/bangla-expert/references/journalism_domains.md,sha256=P8Fkt_hCEB6BhzslVV8oQFo9GtVhc3-nnsJQVpSOVYc,2113
|
|
32
|
+
tools/assets/skills/bangla-expert/references/numerals.md,sha256=PerC1Tco57n8WWdJdXaz7qYgBDC136mJez1iQ5X5keg,3109
|
|
33
|
+
tools/assets/skills/bangla-expert/references/official_letters.md,sha256=tXEMYg0EtuXdIy8heINm1CXATPkGfOZBMJxUUW4uyI8,2751
|
|
34
|
+
tools/assets/skills/bangla-expert/references/orthography.md,sha256=jP5-XurnrU2FiszUDzjRKgujA7xbcjCim4sdeci7Niw,1090
|
|
35
|
+
tools/bn_legal/__init__.py,sha256=4mRhovvTMOtqY_jcoBPW8HcMPbUMbXDXX_5jq8C8dXw,271
|
|
36
|
+
tools/bn_legal/writs.py,sha256=W_mGPyI-8YdwFgTHGzFsticFhuzDevoSTkpB-XgPaQg,11387
|
|
37
|
+
tools/bn_lint/__init__.py,sha256=O-gsTo7dpJJSBG7G4zitiFJSWJtlrgG3ax89o6QV9eU,170
|
|
38
|
+
tools/bn_lint/calques.py,sha256=ACOOEuIAEGU071SZb2t5zfn68anSTisQalLsh0UcxXQ,8906
|
|
39
|
+
tools/bn_lint/guru_chandali.py,sha256=c2Bksr2s2NOIRZflbRPPlfXoSQsqHh3PTqAK9APuEsE,4660
|
|
40
|
+
tools/bn_lint/honorifics.py,sha256=sVsBagNVRGEZUdvpG6QgNnMXmo38rPupQTZ7Jov97DU,16962
|
|
41
|
+
tools/bn_lint/linter.py,sha256=Y7p8VRo0et7Ad69cXTO-f9WjuNYreXPfrGJoTs2L7rA,915
|
|
42
|
+
tools/bn_lint/models.py,sha256=jVgMBVZFnfhTe3bnnFzaDy3raUBtM24fiaX_9mNM_70,335
|
|
43
|
+
tools/bn_normalize/__init__.py,sha256=HzlIa1f5-Gqq-JnjU24ImCtMUqCda2x9qPyiRrOgsV4,638
|
|
44
|
+
tools/bn_normalize/lexicon.py,sha256=OvxMxaOAW8GEMYQR3JwaVxqzyGfaxOlr-cZonuu6S6A,14203
|
|
45
|
+
tools/bn_normalize/normalizer.py,sha256=fIZXnWcMQxpvF9AXdf3lIk_XCHSYzqP7kLPSG94XHCc,1545
|
|
46
|
+
tools/bn_normalize/orthography.py,sha256=GIi19ID5-qyOsJcFV5oM9_KE6N8r3Iv3IwA4YTWYW_0,8751
|
|
47
|
+
tools/bn_normalize/profiles.py,sha256=BQQzcGMrRM2C6T3kby7_3l6tdB20nJVhPRMKlS0VR0Q,1345
|
|
48
|
+
tools/bn_normalize/punctuation.py,sha256=AEdZln-4Qxtw4FRy1_vEMflPtkh5f3QPpVRYun4olwQ,3544
|
|
49
|
+
tools/bn_normalize/repair.py,sha256=ohU7DQe9_g76ux2u5reV8kkDKR5JtrdSWKVkhTeK1G4,15551
|
|
50
|
+
tools/bn_normalize/unicode.py,sha256=k3qGxqv9Tms1q7J_TObzyQwrfd3k7VSvXUtUlJXat_Q,2563
|
|
51
|
+
tools/bn_numbers/__init__.py,sha256=AlhYBcoIt6rtiBna7wnti4THSnKhmoCuvVG8Wi14CRE,734
|
|
52
|
+
tools/bn_numbers/currency.py,sha256=ztYNkJxv9GUU1Mba4nZVAq5XhWm1hmoNc8gF99Mkofw,1754
|
|
53
|
+
tools/bn_numbers/dates.py,sha256=TTWYuO1Q_x3Zho_urwPFTJT7oAuNm2eYZFC3yHTxFvE,1648
|
|
54
|
+
tools/bn_numbers/decimals.py,sha256=sPEcUxEgA2dqCmWnadKnFAGhTbl5pJlHHVsL2dh7d-8,1524
|
|
55
|
+
tools/bn_numbers/digits.py,sha256=qbBqHEDqBmbdUBlgFJ7h0h2IAXWt0Z9tx0FzwlJ2PkQ,1950
|
|
56
|
+
tools/bn_numbers/fractions.py,sha256=F8PUHpL8HcVIlc3cwURftIXMPp9LSAStZEsDDbkR7_8,2347
|
|
57
|
+
tools/bn_numbers/ordinals.py,sha256=-gg4aCUxTCuKON-vAbuSyUF4o4urfGKDF0cW5HwcKqs,1435
|
|
58
|
+
tools/bn_numbers/words.py,sha256=63_tz2SsfX8p4iYMrxrS0t2HD2dooa78ChLjBWofzUU,5656
|
|
59
|
+
tools/bn_prosody/__init__.py,sha256=sMRYWvJCKW5ZqfDyLgZK1U6dqLNUZv_mhwqgx_Aq2Rs,418
|
|
60
|
+
tools/bn_prosody/metre.py,sha256=hpfdXRq5KbYxRdFkadCiXEBnZzi33JyIcYShkn2gIIM,4888
|
|
61
|
+
tools/bn_prosody/syllable.py,sha256=n1nG9WKo6lLaoVoy9UFaZKX0ZftKMbs7GT60iNTxrOY,4856
|
|
62
|
+
tools/mcp_server/__init__.py,sha256=fwHD4DZKCxMBqJAAvdVQi0gVNiUzLaiinelYZJF6RYE,125
|
|
63
|
+
tools/mcp_server/__main__.py,sha256=HGIVnrDGBNp3i2GDDDeyl_ldSz3eFDS652B7pJNLZ_8,121
|
|
64
|
+
tools/mcp_server/server.py,sha256=LsJs0b-hpf5noQiqi5aPBOXtYlfPqLRB-0SfGZzUZ2k,13893
|
|
65
|
+
bhashabid-1.0.0.dist-info/METADATA,sha256=gZsCKMkY3sWcA9kt7Ntm13ZlsSo3OhTo4WgfyOmZre8,28111
|
|
66
|
+
bhashabid-1.0.0.dist-info/WHEEL,sha256=mOdXR9C3wMVov7cSY8PY-XCzo3SmmWCHvGQr5csJoao,91
|
|
67
|
+
bhashabid-1.0.0.dist-info/entry_points.txt,sha256=7tZw5ZCdnsziMVMP85rB01GMhOCDNSZ2wZ1GI_4RkzA,90
|
|
68
|
+
bhashabid-1.0.0.dist-info/top_level.txt,sha256=GtJVrIkKLu4XkaRvdGew3IL6utK3DtkdacTTsx65niI,16
|
|
69
|
+
bhashabid-1.0.0.dist-info/RECORD,,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Md. Ashikur Rahman Bhuiyan and Bhashabid Contributors
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Md. Ashikur Rahman Bhuiyan and Bhashabid Contributors
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
tools/__init__.py
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
# Bhashabid Tools Package
|
|
@@ -0,0 +1,122 @@
|
|
|
1
|
+
# tools/adversarial_generator.py
|
|
2
|
+
# Dynamic Adversarial Perturbation & Multi-Domain Linguistic Stress Generator
|
|
3
|
+
import random
|
|
4
|
+
import re
|
|
5
|
+
from typing import List, Tuple, Dict
|
|
6
|
+
|
|
7
|
+
# Known common LLM corruption patterns across Bengali sectors
|
|
8
|
+
ORTHOGRAPHIC_PERTURBATIONS = [
|
|
9
|
+
("পোস্ট", "পোষ্ট"),
|
|
10
|
+
("মাস্টার", "মাষ্টার"),
|
|
11
|
+
("স্টেশন", "ষ্টেশন"),
|
|
12
|
+
("স্টাফ", "ষ্টাফ"),
|
|
13
|
+
("বাড়ি", "বাড়ী"),
|
|
14
|
+
("শ্রেণি", "শ্রেণী"),
|
|
15
|
+
("পাখি", "পাখী"),
|
|
16
|
+
("বেশি", "বেশী"),
|
|
17
|
+
("ইতিমধ্যে", "ইতোমধ্যে"),
|
|
18
|
+
("পুরস্কার", "পুরষ্কার"),
|
|
19
|
+
("গীতাঞ্জলি", "গীতাঞ্জলী"),
|
|
20
|
+
("শ্রদ্ধাঞ্জলি", "শ্রদ্ধাঞ্জলী"),
|
|
21
|
+
("প্রমিত", "প্রমীত"),
|
|
22
|
+
]
|
|
23
|
+
|
|
24
|
+
SYNTACTIC_CALQUES = [
|
|
25
|
+
("কারণে", "দ্বারা চালিত হয়ে"),
|
|
26
|
+
("পরিচালিত হচ্ছে", "দ্বারা পরিচালিত হইতেছে"),
|
|
27
|
+
("অনুষ্ঠিত হয়", "দ্বারা অনুষ্ঠিত করা হয়"),
|
|
28
|
+
("নেতৃত্বে", "কর্তৃক চালিত হয়ে"),
|
|
29
|
+
("সিদ্ধান্ত গৃহীত হয়", "সিদ্ধান্ত নেওয়া হয়েছে"),
|
|
30
|
+
("সবশেষে", "দিনের শেষে"),
|
|
31
|
+
]
|
|
32
|
+
|
|
33
|
+
SADHU_CHOLITO_INJECTIONS = [
|
|
34
|
+
("হয়েছে", "হইয়াছে"),
|
|
35
|
+
("করেছেন", "করিয়াছেন"),
|
|
36
|
+
("করছে", "করিতেছে"),
|
|
37
|
+
("হবে", "হইবে"),
|
|
38
|
+
("বললেন", "বলিলেন"),
|
|
39
|
+
("যাচ্ছে", "যাইতেছে"),
|
|
40
|
+
]
|
|
41
|
+
|
|
42
|
+
NUMERAL_CORRUPTIONS = [
|
|
43
|
+
("১", "1"),
|
|
44
|
+
("২", "2"),
|
|
45
|
+
("৩", "3"),
|
|
46
|
+
("৪", "4"),
|
|
47
|
+
("৫", "5"),
|
|
48
|
+
("৬", "6"),
|
|
49
|
+
("৭", "7"),
|
|
50
|
+
("৮", "8"),
|
|
51
|
+
("৯", "9"),
|
|
52
|
+
("০", "0"),
|
|
53
|
+
]
|
|
54
|
+
|
|
55
|
+
class AdversarialGenerator:
|
|
56
|
+
"""
|
|
57
|
+
Generates dynamic linguistic perturbations, synthetic edge cases,
|
|
58
|
+
and adversarial sentence variants to ensure continuous learning and
|
|
59
|
+
prevent stagnation on static benchmark data.
|
|
60
|
+
"""
|
|
61
|
+
|
|
62
|
+
def __init__(self, seed: int = None):
|
|
63
|
+
if seed is not None:
|
|
64
|
+
random.seed(seed)
|
|
65
|
+
|
|
66
|
+
def perturb_orthography(self, text: str) -> Tuple[str, str]:
|
|
67
|
+
"""Injects non-standard or pre-1992 spelling variants common in LLMs."""
|
|
68
|
+
for clean, flawed in ORTHOGRAPHIC_PERTURBATIONS:
|
|
69
|
+
if clean in text:
|
|
70
|
+
return text.replace(clean, flawed, 1), f"Orthographic corruption: '{clean}' -> '{flawed}'"
|
|
71
|
+
return text, "No orthographic pattern found"
|
|
72
|
+
|
|
73
|
+
def perturb_calque(self, text: str) -> Tuple[str, str]:
|
|
74
|
+
"""Injects literal English passive syntax calques."""
|
|
75
|
+
for clean, flawed in SYNTACTIC_CALQUES:
|
|
76
|
+
if clean in text:
|
|
77
|
+
return text.replace(clean, flawed, 1), f"Syntactic calque: '{clean}' -> '{flawed}'"
|
|
78
|
+
return text, "No syntactic calque pattern found"
|
|
79
|
+
|
|
80
|
+
def perturb_sadhu_cholito_mix(self, text: str) -> Tuple[str, str]:
|
|
81
|
+
"""Injects Guru-Chondali (Sadhu/Cholito verb clash) defect."""
|
|
82
|
+
for cholito, sadhu in SADHU_CHOLITO_INJECTIONS:
|
|
83
|
+
if cholito in text:
|
|
84
|
+
# Replace only first occurrence so rest remains Cholito, creating a clash
|
|
85
|
+
return text.replace(cholito, sadhu, 1), f"Guru-Chondali clash: injected Sadhu '{sadhu}' into Cholito sentence"
|
|
86
|
+
return text, "No verb clash pattern found"
|
|
87
|
+
|
|
88
|
+
def perturb_unicode(self, text: str) -> Tuple[str, str]:
|
|
89
|
+
"""Injects non-NFC decomposed Bengali Unicode (e.g. split O-kar into E-kar + Aa-kar)."""
|
|
90
|
+
# Bengali O-kar is \u09cb; decomposed is \u09c7\u09be
|
|
91
|
+
if "\u09cb" in text:
|
|
92
|
+
return text.replace("\u09cb", "\u09c7\u09be", 1), "Unicode decomposition: NFC O-kar converted to split [E-kar + Aa-kar]"
|
|
93
|
+
return text, "No decomposable vowel found"
|
|
94
|
+
|
|
95
|
+
def generate_dynamic_batch(self, base_sentences: List[Tuple[str, str, str]], sample_size: int = 25) -> List[Tuple[str, str, str]]:
|
|
96
|
+
"""
|
|
97
|
+
Takes base sentences (domain, text, category) and synthesizes fresh,
|
|
98
|
+
adversarial test cases for practice rounds.
|
|
99
|
+
"""
|
|
100
|
+
if not base_sentences:
|
|
101
|
+
return []
|
|
102
|
+
|
|
103
|
+
selected = random.sample(base_sentences, min(len(base_sentences), sample_size))
|
|
104
|
+
dynamic_batch = []
|
|
105
|
+
|
|
106
|
+
methods = [
|
|
107
|
+
self.perturb_orthography,
|
|
108
|
+
self.perturb_calque,
|
|
109
|
+
self.perturb_sadhu_cholito_mix,
|
|
110
|
+
self.perturb_unicode
|
|
111
|
+
]
|
|
112
|
+
|
|
113
|
+
for domain, text, cat in selected:
|
|
114
|
+
# 50% chance of introducing an adversarial perturbation
|
|
115
|
+
if random.random() > 0.5:
|
|
116
|
+
method = random.choice(methods)
|
|
117
|
+
perturbed_text, label = method(text)
|
|
118
|
+
dynamic_batch.append((domain, perturbed_text, f"DYNAMIC_ADVERSARIAL: {label}"))
|
|
119
|
+
else:
|
|
120
|
+
dynamic_batch.append((domain, text, cat))
|
|
121
|
+
|
|
122
|
+
return dynamic_batch
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
You are an expert native Bengali linguist adhering to Bangla Academy standard orthography.
|
|
2
|
+
|
|
3
|
+
Operating Rules:
|
|
4
|
+
1. Standard Cholito Register: Never mix Sadhu participles (করিয়া, আসিয়া) with Cholito verbs.
|
|
5
|
+
2. Honorific Concordance: Strictly match verbs with pronouns (আপনি -> করেন, তুমি -> করো, তুই -> করিস).
|
|
6
|
+
3. Orthography: Foreign loanwords take 'স'/'স্ট' (স্টেশন, পোস্ট), non-tatsama words take short 'ি' (পাখি, বাড়ি).
|
|
7
|
+
4. Numerals: 0-99 numbers have irregular spellings (৯৯ = নিরানব্বই). Group digits as ১২,৩৪,৫৬৭.
|
|
8
|
+
5. Punctuation: Use Bengali Dari (।) for sentence endings. Preserve decimals (৩.১৪).
|
|
9
|
+
6. Natural Syntax: Translate idioms culturally (ভাবানুবাদ), never literal English calques.
|