lanka-rag 0.1.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- lanka_rag-0.1.1/.gitignore +12 -0
- lanka_rag-0.1.1/CITATION.cff +30 -0
- lanka_rag-0.1.1/LICENSE +21 -0
- lanka_rag-0.1.1/PKG-INFO +296 -0
- lanka_rag-0.1.1/README.md +263 -0
- lanka_rag-0.1.1/pyproject.toml +75 -0
- lanka_rag-0.1.1/src/lanka_rag/__init__.py +3 -0
- lanka_rag-0.1.1/src/lanka_rag/bench/__init__.py +0 -0
- lanka_rag-0.1.1/src/lanka_rag/bench/datasets.py +146 -0
- lanka_rag-0.1.1/src/lanka_rag/bench/metrics.py +73 -0
- lanka_rag-0.1.1/src/lanka_rag/bench/report.py +193 -0
- lanka_rag-0.1.1/src/lanka_rag/bench/run.py +162 -0
- lanka_rag-0.1.1/src/lanka_rag/bm25.py +59 -0
- lanka_rag-0.1.1/src/lanka_rag/cli.py +166 -0
- lanka_rag-0.1.1/src/lanka_rag/embedders.py +133 -0
- lanka_rag-0.1.1/src/lanka_rag/models.py +112 -0
- lanka_rag-0.1.1/src/lanka_rag/rag/__init__.py +0 -0
- lanka_rag-0.1.1/src/lanka_rag/rag/chain.py +95 -0
- lanka_rag-0.1.1/src/lanka_rag/rag/ingest.py +112 -0
- lanka_rag-0.1.1/src/lanka_rag/rag/store.py +150 -0
- lanka_rag-0.1.1/src/lanka_rag/text.py +183 -0
- lanka_rag-0.1.1/tests/test_bench.py +93 -0
- lanka_rag-0.1.1/tests/test_rag.py +76 -0
- lanka_rag-0.1.1/tests/test_text.py +73 -0
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
cff-version: 1.2.0
|
|
2
|
+
message: "If you use this benchmark or its results, please cite it as below."
|
|
3
|
+
title: "lanka-rag: a Sinhala and Tamil embedding retrieval benchmark and local RAG assistant"
|
|
4
|
+
type: software
|
|
5
|
+
authors:
|
|
6
|
+
- given-names: Nipuna
|
|
7
|
+
family-names: Lakruwan
|
|
8
|
+
repository-code: "https://github.com/Nipuna-Lakruwan/lanka-rag"
|
|
9
|
+
license: MIT
|
|
10
|
+
version: 0.1.1
|
|
11
|
+
date-released: "2026-11-16"
|
|
12
|
+
keywords:
|
|
13
|
+
- Sinhala
|
|
14
|
+
- Tamil
|
|
15
|
+
- Singlish
|
|
16
|
+
- Sri Lanka
|
|
17
|
+
- retrieval
|
|
18
|
+
- embeddings
|
|
19
|
+
- benchmark
|
|
20
|
+
- RAG
|
|
21
|
+
references:
|
|
22
|
+
- type: article
|
|
23
|
+
title: "The Belebele Benchmark: a Parallel Reading Comprehension Dataset in 122 Language Variants"
|
|
24
|
+
authors:
|
|
25
|
+
- family-names: Bandarkar
|
|
26
|
+
given-names: Lucas
|
|
27
|
+
- name: "et al."
|
|
28
|
+
year: 2024
|
|
29
|
+
conference:
|
|
30
|
+
name: "ACL 2024"
|
lanka_rag-0.1.1/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Nipuna Lakruwan
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
lanka_rag-0.1.1/PKG-INFO
ADDED
|
@@ -0,0 +1,296 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: lanka-rag
|
|
3
|
+
Version: 0.1.1
|
|
4
|
+
Summary: Local-first Sinhala/Tamil RAG with Ollama + LangChain, and a retrieval benchmark of embedding models on Sinhala, Tamil and Singlish.
|
|
5
|
+
Project-URL: Homepage, https://github.com/Nipuna-Lakruwan/lanka-rag
|
|
6
|
+
Project-URL: Results, https://github.com/Nipuna-Lakruwan/lanka-rag/blob/main/results/results.md
|
|
7
|
+
Project-URL: Issues, https://github.com/Nipuna-Lakruwan/lanka-rag/issues
|
|
8
|
+
Author: Nipuna Lakruwan
|
|
9
|
+
License-Expression: MIT
|
|
10
|
+
License-File: LICENSE
|
|
11
|
+
Keywords: benchmark,embeddings,langchain,low-resource-languages,multilingual,nlp,ollama,rag,retrieval,singlish,sinhala,sri-lanka,tamil
|
|
12
|
+
Classifier: Development Status :: 3 - Alpha
|
|
13
|
+
Classifier: Intended Audience :: Developers
|
|
14
|
+
Classifier: Intended Audience :: Science/Research
|
|
15
|
+
Classifier: Natural Language :: Tamil
|
|
16
|
+
Classifier: Programming Language :: Python :: 3
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
20
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
21
|
+
Classifier: Topic :: Text Processing :: Linguistic
|
|
22
|
+
Requires-Python: >=3.11
|
|
23
|
+
Requires-Dist: httpx>=0.27
|
|
24
|
+
Requires-Dist: langchain-core>=1.0
|
|
25
|
+
Requires-Dist: langchain-ollama>=1.0
|
|
26
|
+
Requires-Dist: langchain-text-splitters>=1.0
|
|
27
|
+
Requires-Dist: numpy>=1.26
|
|
28
|
+
Requires-Dist: pypdf>=5.0
|
|
29
|
+
Requires-Dist: rich>=13.7
|
|
30
|
+
Provides-Extra: hf
|
|
31
|
+
Requires-Dist: sentence-transformers>=3.0; extra == 'hf'
|
|
32
|
+
Description-Content-Type: text/markdown
|
|
33
|
+
|
|
34
|
+
# lanka-rag
|
|
35
|
+
|
|
36
|
+
[](https://github.com/Nipuna-Lakruwan/lanka-rag/actions/workflows/ci.yml)
|
|
37
|
+
[](https://pypi.org/project/lanka-rag/)
|
|
38
|
+
[](https://github.com/Nipuna-Lakruwan/lanka-rag/blob/main/LICENSE)
|
|
39
|
+
[](https://github.com/Nipuna-Lakruwan/lanka-rag/blob/main/pyproject.toml)
|
|
40
|
+
[](https://ollama.com)
|
|
41
|
+
|
|
42
|
+
Sinhala embedding benchmark · Tamil retrieval · Singlish · local RAG for Sri Lanka ·
|
|
43
|
+
සිංහල · தமிழ்
|
|
44
|
+
|
|
45
|
+
**Which embedding model actually understands Sinhala?** This repo has a reproducible retrieval
|
|
46
|
+
benchmark for Sinhala, Tamil and Singlish (romanized Sinhala), plus a local-first RAG assistant
|
|
47
|
+
for Sri Lankan documents built with LangChain and Ollama. Everything runs on a laptop with no
|
|
48
|
+
API keys.
|
|
49
|
+
|
|
50
|
+
## Key findings
|
|
51
|
+
|
|
52
|
+
Eight Ollama embedding models and two BM25 baselines, nine tasks, 900 questions each, on an
|
|
53
|
+
Apple M1 with 8 GB of RAM.
|
|
54
|
+
|
|
55
|
+
1. **Use `bge-m3` for Sinhala and Tamil.** It is best on 7 of 9 tasks, including every
|
|
56
|
+
cross-lingual one (nDCG@10 0.913 Sinhala, 0.900 Tamil, 0.879 English→Sinhala).
|
|
57
|
+
`snowflake-arctic-embed2` is a close second (0.906 Sinhala, a tie on Tamil) and their 95%
|
|
58
|
+
CIs overlap. The two tasks bge-m3 loses, English and Singlish, go to `embeddinggemma`.
|
|
59
|
+
2. **The popular default is near-random on Sinhala.** `nomic-embed-text`, the embedding model in
|
|
60
|
+
many Ollama RAG tutorials, scores 0.078 on Sinhala and 0.142 on Tamil while scoring 0.923 on
|
|
61
|
+
English. A RAG app built on it will quietly fail for Sri Lankan users.
|
|
62
|
+
3. **Character n-gram BM25 almost matches the best model within one language.** 0.899 vs 0.913
|
|
63
|
+
on Sinhala, with no model at all. It beats every dense model except bge-m3 and snowflake.
|
|
64
|
+
Word-level BM25 is 14 points worse on Sinhala and 18 on Tamil, because Sinhala and Tamil
|
|
65
|
+
attach case markers and postpositions to words. Lexical search only fails across languages,
|
|
66
|
+
which is why the RAG app uses both (hybrid retrieval).
|
|
67
|
+
4. **A strong English score says nothing about Sinhala.** `qwen3-embedding:0.6b` scores 0.956
|
|
68
|
+
on English and 0.410 on Sinhala. It is also the slowest model here on Sinhala (0.65 passages/s
|
|
69
|
+
vs 9 for bge-m3), because its tokenizer spends up to ~2,000 tokens on a 75-word passage.
|
|
70
|
+
5. **Nobody can search Sinhala in Singlish yet.** The best model, `embeddinggemma`, gets 0.309
|
|
71
|
+
nDCG@10 (21% Recall@1) for romanized questions. bge-m3 gets 0.154. This is the most useful
|
|
72
|
+
gap to close, since it is how most people type Sinhala on a phone.
|
|
73
|
+
6. **Typing without zero width joiners costs dense models about 1 point.** Small but consistent
|
|
74
|
+
(bge-m3 0.913 → 0.906, paraphrase-multilingual 0.557 → 0.536). BM25 is unaffected because the
|
|
75
|
+
tokenizer strips ZWJ.
|
|
76
|
+
|
|
77
|
+
Two notes on fairness: `paraphrase-multilingual` reads at most 128 tokens, so it sees only the
|
|
78
|
+
start of long Sinhala passages, and the speed column was measured with nothing else running on
|
|
79
|
+
the machine. No model hit its context limit on these passages (checked in the Ollama logs).
|
|
80
|
+
|
|
81
|
+
<picture>
|
|
82
|
+
<source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/Nipuna-Lakruwan/lanka-rag/main/results/ndcg_heatmap_dark.svg">
|
|
83
|
+
<img alt="Heatmap of nDCG@10 for each embedding model on each Sinhala/Tamil retrieval task" src="https://raw.githubusercontent.com/Nipuna-Lakruwan/lanka-rag/main/results/ndcg_heatmap_light.svg">
|
|
84
|
+
</picture>
|
|
85
|
+
|
|
86
|
+
## Results
|
|
87
|
+
|
|
88
|
+
<!-- RESULTS:START -->
|
|
89
|
+
**nDCG@10** (higher is better; best per column in bold). "Sinhala avg" averages every task with Sinhala on either side.
|
|
90
|
+
|
|
91
|
+
| Model | Params | Sinhala avg | si→si | si→si (no ZWJ) | si-Latn→si | ta→ta | en→en | en→si | si→en | ta→si | si→ta |
|
|
92
|
+
|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|
|
|
93
|
+
| bge-m3 | 568M | 0.757 | **0.913** | **0.906** | 0.154 | **0.900** | 0.955 | **0.879** | **0.835** | **0.802** | **0.811** |
|
|
94
|
+
| snowflake-arctic-embed2 | 568M | 0.739 | 0.906 | 0.897 | 0.143 | 0.900 | 0.945 | 0.852 | 0.794 | 0.784 | 0.797 |
|
|
95
|
+
| granite-embedding | 278M | 0.616 | 0.814 | 0.806 | 0.092 | 0.792 | 0.931 | 0.752 | 0.668 | 0.578 | 0.607 |
|
|
96
|
+
| embeddinggemma | 308M | 0.613 | 0.718 | 0.706 | **0.309** | 0.868 | **0.965** | 0.790 | 0.567 | 0.664 | 0.539 |
|
|
97
|
+
| paraphrase-multilingual | 278M | 0.446 | 0.557 | 0.536 | 0.022 | 0.493 | 0.885 | 0.667 | 0.489 | 0.433 | 0.420 |
|
|
98
|
+
| bm25-char | - | 0.295 | 0.899 | 0.899 | 0.046 | 0.863 | 0.888 | 0.052 | 0.063 | 0.053 | 0.056 |
|
|
99
|
+
| bm25-word | - | 0.254 | 0.755 | 0.755 | 0.046 | 0.680 | 0.879 | 0.059 | 0.062 | 0.051 | 0.053 |
|
|
100
|
+
| qwen3-embedding-0.6b | 0.6B | 0.218 | 0.410 | 0.392 | 0.049 | 0.697 | 0.956 | 0.291 | 0.121 | 0.147 | 0.114 |
|
|
101
|
+
| nomic-embed-text † | 137M | 0.059 | 0.078 | 0.078 | 0.040 | 0.142 | 0.923 | 0.075 | 0.052 | 0.050 | 0.041 |
|
|
102
|
+
|
|
103
|
+
† English-centric model, included as a control.
|
|
104
|
+
|
|
105
|
+
**Recall@1 / Recall@10** on the three monolingual tasks, plus 95% bootstrap CI of nDCG@10 for si→si:
|
|
106
|
+
|
|
107
|
+
| Model | si→si R@1 | si→si R@10 | si→si nDCG@10 95% CI | ta→ta R@1 | Singlish→si R@1 | Embed speed (passages/s) |
|
|
108
|
+
|---|---:|---:|---:|---:|---:|---:|
|
|
109
|
+
| bge-m3 | 0.851 | 0.966 | 0.896–0.928 | 0.838 | 0.099 | 9.25 |
|
|
110
|
+
| snowflake-arctic-embed2 | 0.850 | 0.958 | 0.890–0.921 | 0.839 | 0.087 | 7.49 |
|
|
111
|
+
| granite-embedding | 0.716 | 0.908 | 0.793–0.835 | 0.701 | 0.056 | 13.86 |
|
|
112
|
+
| embeddinggemma | 0.621 | 0.818 | 0.693–0.745 | 0.794 | 0.213 | 8.54 |
|
|
113
|
+
| paraphrase-multilingual | 0.417 | 0.717 | 0.531–0.585 | 0.366 | 0.004 | 27.33 |
|
|
114
|
+
| bm25-char | 0.837 | 0.952 | 0.883–0.915 | 0.797 | 0.032 | - |
|
|
115
|
+
| bm25-word | 0.669 | 0.842 | 0.729–0.780 | 0.592 | 0.038 | - |
|
|
116
|
+
| qwen3-embedding-0.6b | 0.278 | 0.553 | 0.382–0.436 | 0.603 | 0.030 | 0.65 |
|
|
117
|
+
| nomic-embed-text | 0.058 | 0.107 | 0.063–0.095 | 0.092 | 0.026 | 30.7 |
|
|
118
|
+
<!-- RESULTS:END -->
|
|
119
|
+
|
|
120
|
+
Raw per-query ranks for every model are in [`results/raw/`](https://github.com/Nipuna-Lakruwan/lanka-rag/tree/main/results/raw), so you can run your
|
|
121
|
+
own error analysis or significance tests.
|
|
122
|
+
|
|
123
|
+
## The benchmark
|
|
124
|
+
|
|
125
|
+
Sinhala has very little retrieval evaluation data. [Belebele](https://huggingface.co/datasets/facebook/belebele)
|
|
126
|
+
(Meta, CC BY-SA 4.0) has 900 human-written questions over 488 passages, and the *same*
|
|
127
|
+
questions and passages exist in Sinhala script, romanized Sinhala, Tamil and English. Since every
|
|
128
|
+
language is aligned by passage, one dataset gives nine tasks with identical relevance labels:
|
|
129
|
+
|
|
130
|
+
| Task | Query | Passages | What it tells you |
|
|
131
|
+
|---|---|---|---|
|
|
132
|
+
| `si→si` | Sinhala | Sinhala | Basic Sinhala retrieval |
|
|
133
|
+
| `si→si (no ZWJ)` | Sinhala, zero width joiners removed | Sinhala | Robustness to how people actually type (`ශ්රී` vs `ශ්රී`). Many phone keyboards drop the ZWJ |
|
|
134
|
+
| `si-Latn→si` | Singlish | Sinhala | Can users search Sinhala documents the way they text? |
|
|
135
|
+
| `ta→ta` | Tamil | Tamil | Basic Tamil retrieval |
|
|
136
|
+
| `en→en` | English | English | Reference: is the model any good at all? |
|
|
137
|
+
| `en→si`, `si→en` | | | Cross-lingual, the common case for bilingual Sri Lankan documents |
|
|
138
|
+
| `ta→si`, `si→ta` | | | Tamil ↔ Sinhala without going through English |
|
|
139
|
+
|
|
140
|
+
Each query has exactly one relevant passage. Metrics: nDCG@10 (main), MRR@10 and Recall@1/5/10,
|
|
141
|
+
with 95% bootstrap confidence intervals.
|
|
142
|
+
|
|
143
|
+
MTEB already has a `BelebeleRetrieval` task. This benchmark adds the Singlish, ZWJ-robustness and
|
|
144
|
+
Tamil↔Sinhala tasks, local models you can actually run through Ollama, the exact prompt
|
|
145
|
+
format each model expects ([`models.py`](https://github.com/Nipuna-Lakruwan/lanka-rag/blob/main/src/lanka_rag/models.py)), and lexical baselines that
|
|
146
|
+
handle Sinhala correctly.
|
|
147
|
+
|
|
148
|
+
### Sinhala text pitfalls this code handles
|
|
149
|
+
|
|
150
|
+
- **`\w` regexes shred Sinhala words.** Vowel signs are combining marks, which Python's `\w`
|
|
151
|
+
doesn't count as word characters: `re.findall(r"\w+", "ජනාධිපතිවරයා")` returns
|
|
152
|
+
`['ජන', 'ධ', 'පත', 'වරය']`. [`text.tokenize`](https://github.com/Nipuna-Lakruwan/lanka-rag/blob/main/src/lanka_rag/text.py) splits on
|
|
153
|
+
Unicode categories instead, so a naive BM25 baseline here is not secretly broken.
|
|
154
|
+
- **ZWJ spellings.** `ශ්රී` (with U+200D) and `ශ්රී` are the same word. Lexical matching strips
|
|
155
|
+
ZWJ, and stray joiners that don't follow the virama are removed everywhere.
|
|
156
|
+
- **Two encodings of one vowel sign.** `ො` can be stored as U+0DDC or as `ෙ` + `ා`. Everything is NFC
|
|
157
|
+
normalized.
|
|
158
|
+
- **Legacy-font PDFs.** Many government PDFs use pre-Unicode fonts (FM Abhaya, DL-Manel), so the
|
|
159
|
+
extracted "text" looks like `Y%S ,xld`. `lanka-rag inspect file.pdf` flags these pages, and
|
|
160
|
+
ingestion skips them so garbage never reaches the vector store.
|
|
161
|
+
|
|
162
|
+
## Quick start
|
|
163
|
+
|
|
164
|
+
Needs [Ollama](https://ollama.com). To use the CLI and the library, install from PyPI:
|
|
165
|
+
|
|
166
|
+
```bash
|
|
167
|
+
pip install lanka-rag
|
|
168
|
+
```
|
|
169
|
+
|
|
170
|
+
To reproduce the benchmark or run the Acts demo, clone the repo and use
|
|
171
|
+
[uv](https://docs.astral.sh/uv/):
|
|
172
|
+
|
|
173
|
+
```bash
|
|
174
|
+
git clone https://github.com/Nipuna-Lakruwan/lanka-rag.git
|
|
175
|
+
cd lanka-rag
|
|
176
|
+
uv sync
|
|
177
|
+
```
|
|
178
|
+
|
|
179
|
+
### Reproduce the benchmark
|
|
180
|
+
|
|
181
|
+
```bash
|
|
182
|
+
uv run lanka-rag bench download
|
|
183
|
+
```
|
|
184
|
+
|
|
185
|
+
```bash
|
|
186
|
+
uv run lanka-rag bench run bm25-word bm25-char bge-m3 embeddinggemma --pull
|
|
187
|
+
```
|
|
188
|
+
|
|
189
|
+
```bash
|
|
190
|
+
uv run lanka-rag bench report
|
|
191
|
+
```
|
|
192
|
+
|
|
193
|
+
`--pull` downloads missing Ollama models and `--rm-after` deletes them again after scoring
|
|
194
|
+
(handy on a small disk). Embeddings are cached in `.cache/`, so you can rerun the report for free.
|
|
195
|
+
`uv run lanka-rag bench models` lists every model and the prompts it uses. Hugging Face models
|
|
196
|
+
(multilingual-e5, LaBSE) need `uv sync --extra hf`.
|
|
197
|
+
|
|
198
|
+
### Ask questions over your own documents
|
|
199
|
+
|
|
200
|
+
The demo corpus is 1,010 hand-corrected pages of Sri Lankan Acts (1981–2019) in Sinhala, from
|
|
201
|
+
[sinhala-ocr-lk-acts-1010](https://huggingface.co/datasets/avishadilhara/sinhala-ocr-lk-acts-1010)
|
|
202
|
+
(CC BY 4.0). Drop your own PDFs, `.txt`, `.md` or `.html` files into `docs/` the same way.
|
|
203
|
+
|
|
204
|
+
```bash
|
|
205
|
+
uv run python scripts/fetch_acts_demo.py
|
|
206
|
+
```
|
|
207
|
+
|
|
208
|
+
```bash
|
|
209
|
+
uv run lanka-rag ingest docs/acts --embedding bge-m3
|
|
210
|
+
```
|
|
211
|
+
|
|
212
|
+
```bash
|
|
213
|
+
uv run lanka-rag ask "තුරුසවිය අරමුදලේ පරමාර්ථ මොනවාද?"
|
|
214
|
+
```
|
|
215
|
+
|
|
216
|
+
```bash
|
|
217
|
+
uv run lanka-rag chat
|
|
218
|
+
```
|
|
219
|
+
|
|
220
|
+
Retrieval is hybrid: dense embeddings plus character n-gram BM25, fused with reciprocal rank
|
|
221
|
+
fusion ([`rag/store.py`](https://github.com/Nipuna-Lakruwan/lanka-rag/blob/main/src/lanka_rag/rag/store.py)). BM25 only scores chunks in the
|
|
222
|
+
question's language, because across languages it matches on numbers and stray Latin letters and
|
|
223
|
+
adds noise. The answer comes from a local Ollama chat
|
|
224
|
+
model (default `qwen3:4b-instruct-2507-q4_K_M`, change it with `--llm`). It answers in the
|
|
225
|
+
language of the question and cites the passages it used.
|
|
226
|
+
|
|
227
|
+
### Docker
|
|
228
|
+
|
|
229
|
+
```bash
|
|
230
|
+
docker compose up -d ollama
|
|
231
|
+
```
|
|
232
|
+
|
|
233
|
+
```bash
|
|
234
|
+
docker compose exec ollama ollama pull bge-m3
|
|
235
|
+
```
|
|
236
|
+
|
|
237
|
+
```bash
|
|
238
|
+
docker compose run --rm app ingest /docs
|
|
239
|
+
```
|
|
240
|
+
|
|
241
|
+
## Layout
|
|
242
|
+
|
|
243
|
+
```
|
|
244
|
+
src/lanka_rag/
|
|
245
|
+
text.py Sinhala/Tamil normalization, tokenizer, char n-grams, legacy-font detector
|
|
246
|
+
models.py embedding model registry + per-model query/document prompts
|
|
247
|
+
embedders.py Ollama / Hugging Face embedders with an on-disk cache
|
|
248
|
+
bm25.py BM25 over Unicode-aware tokens or char n-grams
|
|
249
|
+
bench/ Belebele tasks, metrics, runner, report + heatmap
|
|
250
|
+
rag/ ingestion, hybrid LangChain retriever, LCEL answer chain
|
|
251
|
+
scripts/ demo corpus download
|
|
252
|
+
tests/ unit tests (no network or models needed)
|
|
253
|
+
```
|
|
254
|
+
|
|
255
|
+
## Limitations
|
|
256
|
+
|
|
257
|
+
- Belebele passages are professional translations of English Wikinews, Wikivoyage and Wikibooks
|
|
258
|
+
text (FLORES-200), not native Sri Lankan writing. Expect real gazettes and news to be harder.
|
|
259
|
+
- 488 passages is a small corpus. Absolute scores are higher than on a real document collection,
|
|
260
|
+
so read the table as a ranking of models.
|
|
261
|
+
- Many questions start with "according to the passage" (`ඡේදයට අනුව`), which adds the same noise
|
|
262
|
+
for every model.
|
|
263
|
+
- This measures retrieval only. Answer quality of the RAG chain is not benchmarked yet, and the
|
|
264
|
+
small default chat model is the weak link. Asked in English whether the Trust Fund Act lets the
|
|
265
|
+
board invest money not immediately needed, retrieval puts the exact clause at rank 2, but
|
|
266
|
+
`qwen3:4b` still replies that it could not find the answer. Same-language questions
|
|
267
|
+
(Sinhala question over Sinhala Acts) work much better. A bigger model (`--llm`) helps if you
|
|
268
|
+
have the RAM.
|
|
269
|
+
- Sinhala is expensive for LLMs: about 1.3 tokens per character in Qwen's tokenizer, so four
|
|
270
|
+
800-character chunks already make a ~4k token prompt. On an 8 GB M1 an answer takes 1–2.5
|
|
271
|
+
minutes.
|
|
272
|
+
|
|
273
|
+
## Roadmap
|
|
274
|
+
|
|
275
|
+
- [ ] FM Abhaya / DL-Manel → Unicode converter so legacy-font gazettes can be indexed
|
|
276
|
+
- [ ] Retrieval benchmark on the Sri Lankan Acts corpus with human-checked Sinhala questions
|
|
277
|
+
- [ ] Answer-quality evaluation of local LLMs (qwen3, gemma3) in Sinhala and Tamil
|
|
278
|
+
- [ ] multilingual-e5 and LaBSE results (code is in, not run yet)
|
|
279
|
+
- [ ] Small web UI
|
|
280
|
+
|
|
281
|
+
## Contributing
|
|
282
|
+
|
|
283
|
+
New models, Sinhala/Tamil test sets and bug reports are welcome. The quickest useful
|
|
284
|
+
contribution is benchmarking a model that is not in the table: see
|
|
285
|
+
[CONTRIBUTING.md](https://github.com/Nipuna-Lakruwan/lanka-rag/blob/main/CONTRIBUTING.md), or [open a model request](https://github.com/Nipuna-Lakruwan/lanka-rag/issues/new?template=model_request.yml).
|
|
286
|
+
|
|
287
|
+
## Citing
|
|
288
|
+
|
|
289
|
+
If you use the benchmark or its results, please cite this repository (GitHub's "Cite this
|
|
290
|
+
repository" button uses [CITATION.cff](https://github.com/Nipuna-Lakruwan/lanka-rag/blob/main/CITATION.cff)) and the Belebele paper.
|
|
291
|
+
|
|
292
|
+
## Credits and licenses
|
|
293
|
+
|
|
294
|
+
Code: MIT. Belebele: CC BY-SA 4.0 (Bandarkar et al., 2023), downloaded at runtime and not
|
|
295
|
+
redistributed. Sri Lankan Acts transcriptions: CC BY 4.0 (avishadilhara). Results files in
|
|
296
|
+
`results/` contain only scores and passage ids.
|
|
@@ -0,0 +1,263 @@
|
|
|
1
|
+
# lanka-rag
|
|
2
|
+
|
|
3
|
+
[](https://github.com/Nipuna-Lakruwan/lanka-rag/actions/workflows/ci.yml)
|
|
4
|
+
[](https://pypi.org/project/lanka-rag/)
|
|
5
|
+
[](https://github.com/Nipuna-Lakruwan/lanka-rag/blob/main/LICENSE)
|
|
6
|
+
[](https://github.com/Nipuna-Lakruwan/lanka-rag/blob/main/pyproject.toml)
|
|
7
|
+
[](https://ollama.com)
|
|
8
|
+
|
|
9
|
+
Sinhala embedding benchmark · Tamil retrieval · Singlish · local RAG for Sri Lanka ·
|
|
10
|
+
සිංහල · தமிழ்
|
|
11
|
+
|
|
12
|
+
**Which embedding model actually understands Sinhala?** This repo has a reproducible retrieval
|
|
13
|
+
benchmark for Sinhala, Tamil and Singlish (romanized Sinhala), plus a local-first RAG assistant
|
|
14
|
+
for Sri Lankan documents built with LangChain and Ollama. Everything runs on a laptop with no
|
|
15
|
+
API keys.
|
|
16
|
+
|
|
17
|
+
## Key findings
|
|
18
|
+
|
|
19
|
+
Eight Ollama embedding models and two BM25 baselines, nine tasks, 900 questions each, on an
|
|
20
|
+
Apple M1 with 8 GB of RAM.
|
|
21
|
+
|
|
22
|
+
1. **Use `bge-m3` for Sinhala and Tamil.** It is best on 7 of 9 tasks, including every
|
|
23
|
+
cross-lingual one (nDCG@10 0.913 Sinhala, 0.900 Tamil, 0.879 English→Sinhala).
|
|
24
|
+
`snowflake-arctic-embed2` is a close second (0.906 Sinhala, a tie on Tamil) and their 95%
|
|
25
|
+
CIs overlap. The two tasks bge-m3 loses, English and Singlish, go to `embeddinggemma`.
|
|
26
|
+
2. **The popular default is near-random on Sinhala.** `nomic-embed-text`, the embedding model in
|
|
27
|
+
many Ollama RAG tutorials, scores 0.078 on Sinhala and 0.142 on Tamil while scoring 0.923 on
|
|
28
|
+
English. A RAG app built on it will quietly fail for Sri Lankan users.
|
|
29
|
+
3. **Character n-gram BM25 almost matches the best model within one language.** 0.899 vs 0.913
|
|
30
|
+
on Sinhala, with no model at all. It beats every dense model except bge-m3 and snowflake.
|
|
31
|
+
Word-level BM25 is 14 points worse on Sinhala and 18 on Tamil, because Sinhala and Tamil
|
|
32
|
+
attach case markers and postpositions to words. Lexical search only fails across languages,
|
|
33
|
+
which is why the RAG app uses both (hybrid retrieval).
|
|
34
|
+
4. **A strong English score says nothing about Sinhala.** `qwen3-embedding:0.6b` scores 0.956
|
|
35
|
+
on English and 0.410 on Sinhala. It is also the slowest model here on Sinhala (0.65 passages/s
|
|
36
|
+
vs 9 for bge-m3), because its tokenizer spends up to ~2,000 tokens on a 75-word passage.
|
|
37
|
+
5. **Nobody can search Sinhala in Singlish yet.** The best model, `embeddinggemma`, gets 0.309
|
|
38
|
+
nDCG@10 (21% Recall@1) for romanized questions. bge-m3 gets 0.154. This is the most useful
|
|
39
|
+
gap to close, since it is how most people type Sinhala on a phone.
|
|
40
|
+
6. **Typing without zero width joiners costs dense models about 1 point.** Small but consistent
|
|
41
|
+
(bge-m3 0.913 → 0.906, paraphrase-multilingual 0.557 → 0.536). BM25 is unaffected because the
|
|
42
|
+
tokenizer strips ZWJ.
|
|
43
|
+
|
|
44
|
+
Two notes on fairness: `paraphrase-multilingual` reads at most 128 tokens, so it sees only the
|
|
45
|
+
start of long Sinhala passages, and the speed column was measured with nothing else running on
|
|
46
|
+
the machine. No model hit its context limit on these passages (checked in the Ollama logs).
|
|
47
|
+
|
|
48
|
+
<picture>
|
|
49
|
+
<source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/Nipuna-Lakruwan/lanka-rag/main/results/ndcg_heatmap_dark.svg">
|
|
50
|
+
<img alt="Heatmap of nDCG@10 for each embedding model on each Sinhala/Tamil retrieval task" src="https://raw.githubusercontent.com/Nipuna-Lakruwan/lanka-rag/main/results/ndcg_heatmap_light.svg">
|
|
51
|
+
</picture>
|
|
52
|
+
|
|
53
|
+
## Results
|
|
54
|
+
|
|
55
|
+
<!-- RESULTS:START -->
|
|
56
|
+
**nDCG@10** (higher is better; best per column in bold). "Sinhala avg" averages every task with Sinhala on either side.
|
|
57
|
+
|
|
58
|
+
| Model | Params | Sinhala avg | si→si | si→si (no ZWJ) | si-Latn→si | ta→ta | en→en | en→si | si→en | ta→si | si→ta |
|
|
59
|
+
|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|
|
|
60
|
+
| bge-m3 | 568M | 0.757 | **0.913** | **0.906** | 0.154 | **0.900** | 0.955 | **0.879** | **0.835** | **0.802** | **0.811** |
|
|
61
|
+
| snowflake-arctic-embed2 | 568M | 0.739 | 0.906 | 0.897 | 0.143 | 0.900 | 0.945 | 0.852 | 0.794 | 0.784 | 0.797 |
|
|
62
|
+
| granite-embedding | 278M | 0.616 | 0.814 | 0.806 | 0.092 | 0.792 | 0.931 | 0.752 | 0.668 | 0.578 | 0.607 |
|
|
63
|
+
| embeddinggemma | 308M | 0.613 | 0.718 | 0.706 | **0.309** | 0.868 | **0.965** | 0.790 | 0.567 | 0.664 | 0.539 |
|
|
64
|
+
| paraphrase-multilingual | 278M | 0.446 | 0.557 | 0.536 | 0.022 | 0.493 | 0.885 | 0.667 | 0.489 | 0.433 | 0.420 |
|
|
65
|
+
| bm25-char | - | 0.295 | 0.899 | 0.899 | 0.046 | 0.863 | 0.888 | 0.052 | 0.063 | 0.053 | 0.056 |
|
|
66
|
+
| bm25-word | - | 0.254 | 0.755 | 0.755 | 0.046 | 0.680 | 0.879 | 0.059 | 0.062 | 0.051 | 0.053 |
|
|
67
|
+
| qwen3-embedding-0.6b | 0.6B | 0.218 | 0.410 | 0.392 | 0.049 | 0.697 | 0.956 | 0.291 | 0.121 | 0.147 | 0.114 |
|
|
68
|
+
| nomic-embed-text † | 137M | 0.059 | 0.078 | 0.078 | 0.040 | 0.142 | 0.923 | 0.075 | 0.052 | 0.050 | 0.041 |
|
|
69
|
+
|
|
70
|
+
† English-centric model, included as a control.
|
|
71
|
+
|
|
72
|
+
**Recall@1 / Recall@10** on the three monolingual tasks, plus 95% bootstrap CI of nDCG@10 for si→si:
|
|
73
|
+
|
|
74
|
+
| Model | si→si R@1 | si→si R@10 | si→si nDCG@10 95% CI | ta→ta R@1 | Singlish→si R@1 | Embed speed (passages/s) |
|
|
75
|
+
|---|---:|---:|---:|---:|---:|---:|
|
|
76
|
+
| bge-m3 | 0.851 | 0.966 | 0.896–0.928 | 0.838 | 0.099 | 9.25 |
|
|
77
|
+
| snowflake-arctic-embed2 | 0.850 | 0.958 | 0.890–0.921 | 0.839 | 0.087 | 7.49 |
|
|
78
|
+
| granite-embedding | 0.716 | 0.908 | 0.793–0.835 | 0.701 | 0.056 | 13.86 |
|
|
79
|
+
| embeddinggemma | 0.621 | 0.818 | 0.693–0.745 | 0.794 | 0.213 | 8.54 |
|
|
80
|
+
| paraphrase-multilingual | 0.417 | 0.717 | 0.531–0.585 | 0.366 | 0.004 | 27.33 |
|
|
81
|
+
| bm25-char | 0.837 | 0.952 | 0.883–0.915 | 0.797 | 0.032 | - |
|
|
82
|
+
| bm25-word | 0.669 | 0.842 | 0.729–0.780 | 0.592 | 0.038 | - |
|
|
83
|
+
| qwen3-embedding-0.6b | 0.278 | 0.553 | 0.382–0.436 | 0.603 | 0.030 | 0.65 |
|
|
84
|
+
| nomic-embed-text | 0.058 | 0.107 | 0.063–0.095 | 0.092 | 0.026 | 30.7 |
|
|
85
|
+
<!-- RESULTS:END -->
|
|
86
|
+
|
|
87
|
+
Raw per-query ranks for every model are in [`results/raw/`](https://github.com/Nipuna-Lakruwan/lanka-rag/tree/main/results/raw), so you can run your
|
|
88
|
+
own error analysis or significance tests.
|
|
89
|
+
|
|
90
|
+
## The benchmark
|
|
91
|
+
|
|
92
|
+
Sinhala has very little retrieval evaluation data. [Belebele](https://huggingface.co/datasets/facebook/belebele)
|
|
93
|
+
(Meta, CC BY-SA 4.0) has 900 human-written questions over 488 passages, and the *same*
|
|
94
|
+
questions and passages exist in Sinhala script, romanized Sinhala, Tamil and English. Since every
|
|
95
|
+
language is aligned by passage, one dataset gives nine tasks with identical relevance labels:
|
|
96
|
+
|
|
97
|
+
| Task | Query | Passages | What it tells you |
|
|
98
|
+
|---|---|---|---|
|
|
99
|
+
| `si→si` | Sinhala | Sinhala | Basic Sinhala retrieval |
|
|
100
|
+
| `si→si (no ZWJ)` | Sinhala, zero width joiners removed | Sinhala | Robustness to how people actually type (`ශ්රී` vs `ශ්රී`). Many phone keyboards drop the ZWJ |
|
|
101
|
+
| `si-Latn→si` | Singlish | Sinhala | Can users search Sinhala documents the way they text? |
|
|
102
|
+
| `ta→ta` | Tamil | Tamil | Basic Tamil retrieval |
|
|
103
|
+
| `en→en` | English | English | Reference: is the model any good at all? |
|
|
104
|
+
| `en→si`, `si→en` | | | Cross-lingual, the common case for bilingual Sri Lankan documents |
|
|
105
|
+
| `ta→si`, `si→ta` | | | Tamil ↔ Sinhala without going through English |
|
|
106
|
+
|
|
107
|
+
Each query has exactly one relevant passage. Metrics: nDCG@10 (main), MRR@10 and Recall@1/5/10,
|
|
108
|
+
with 95% bootstrap confidence intervals.
|
|
109
|
+
|
|
110
|
+
MTEB already has a `BelebeleRetrieval` task. This benchmark adds the Singlish, ZWJ-robustness and
|
|
111
|
+
Tamil↔Sinhala tasks, local models you can actually run through Ollama, the exact prompt
|
|
112
|
+
format each model expects ([`models.py`](https://github.com/Nipuna-Lakruwan/lanka-rag/blob/main/src/lanka_rag/models.py)), and lexical baselines that
|
|
113
|
+
handle Sinhala correctly.
|
|
114
|
+
|
|
115
|
+
### Sinhala text pitfalls this code handles
|
|
116
|
+
|
|
117
|
+
- **`\w` regexes shred Sinhala words.** Vowel signs are combining marks, which Python's `\w`
|
|
118
|
+
doesn't count as word characters: `re.findall(r"\w+", "ජනාධිපතිවරයා")` returns
|
|
119
|
+
`['ජන', 'ධ', 'පත', 'වරය']`. [`text.tokenize`](https://github.com/Nipuna-Lakruwan/lanka-rag/blob/main/src/lanka_rag/text.py) splits on
|
|
120
|
+
Unicode categories instead, so a naive BM25 baseline here is not secretly broken.
|
|
121
|
+
- **ZWJ spellings.** `ශ්රී` (with U+200D) and `ශ්රී` are the same word. Lexical matching strips
|
|
122
|
+
ZWJ, and stray joiners that don't follow the virama are removed everywhere.
|
|
123
|
+
- **Two encodings of one vowel sign.** `ො` can be stored as U+0DDC or as `ෙ` + `ා`. Everything is NFC
|
|
124
|
+
normalized.
|
|
125
|
+
- **Legacy-font PDFs.** Many government PDFs use pre-Unicode fonts (FM Abhaya, DL-Manel), so the
|
|
126
|
+
extracted "text" looks like `Y%S ,xld`. `lanka-rag inspect file.pdf` flags these pages, and
|
|
127
|
+
ingestion skips them so garbage never reaches the vector store.
|
|
128
|
+
|
|
129
|
+
## Quick start
|
|
130
|
+
|
|
131
|
+
Needs [Ollama](https://ollama.com). To use the CLI and the library, install from PyPI:
|
|
132
|
+
|
|
133
|
+
```bash
|
|
134
|
+
pip install lanka-rag
|
|
135
|
+
```
|
|
136
|
+
|
|
137
|
+
To reproduce the benchmark or run the Acts demo, clone the repo and use
|
|
138
|
+
[uv](https://docs.astral.sh/uv/):
|
|
139
|
+
|
|
140
|
+
```bash
|
|
141
|
+
git clone https://github.com/Nipuna-Lakruwan/lanka-rag.git
|
|
142
|
+
cd lanka-rag
|
|
143
|
+
uv sync
|
|
144
|
+
```
|
|
145
|
+
|
|
146
|
+
### Reproduce the benchmark
|
|
147
|
+
|
|
148
|
+
```bash
|
|
149
|
+
uv run lanka-rag bench download
|
|
150
|
+
```
|
|
151
|
+
|
|
152
|
+
```bash
|
|
153
|
+
uv run lanka-rag bench run bm25-word bm25-char bge-m3 embeddinggemma --pull
|
|
154
|
+
```
|
|
155
|
+
|
|
156
|
+
```bash
|
|
157
|
+
uv run lanka-rag bench report
|
|
158
|
+
```
|
|
159
|
+
|
|
160
|
+
`--pull` downloads missing Ollama models and `--rm-after` deletes them again after scoring
|
|
161
|
+
(handy on a small disk). Embeddings are cached in `.cache/`, so you can rerun the report for free.
|
|
162
|
+
`uv run lanka-rag bench models` lists every model and the prompts it uses. Hugging Face models
|
|
163
|
+
(multilingual-e5, LaBSE) need `uv sync --extra hf`.
|
|
164
|
+
|
|
165
|
+
### Ask questions over your own documents
|
|
166
|
+
|
|
167
|
+
The demo corpus is 1,010 hand-corrected pages of Sri Lankan Acts (1981–2019) in Sinhala, from
|
|
168
|
+
[sinhala-ocr-lk-acts-1010](https://huggingface.co/datasets/avishadilhara/sinhala-ocr-lk-acts-1010)
|
|
169
|
+
(CC BY 4.0). Drop your own PDFs, `.txt`, `.md` or `.html` files into `docs/` the same way.
|
|
170
|
+
|
|
171
|
+
```bash
|
|
172
|
+
uv run python scripts/fetch_acts_demo.py
|
|
173
|
+
```
|
|
174
|
+
|
|
175
|
+
```bash
|
|
176
|
+
uv run lanka-rag ingest docs/acts --embedding bge-m3
|
|
177
|
+
```
|
|
178
|
+
|
|
179
|
+
```bash
|
|
180
|
+
uv run lanka-rag ask "තුරුසවිය අරමුදලේ පරමාර්ථ මොනවාද?"
|
|
181
|
+
```
|
|
182
|
+
|
|
183
|
+
```bash
|
|
184
|
+
uv run lanka-rag chat
|
|
185
|
+
```
|
|
186
|
+
|
|
187
|
+
Retrieval is hybrid: dense embeddings plus character n-gram BM25, fused with reciprocal rank
|
|
188
|
+
fusion ([`rag/store.py`](https://github.com/Nipuna-Lakruwan/lanka-rag/blob/main/src/lanka_rag/rag/store.py)). BM25 only scores chunks in the
|
|
189
|
+
question's language, because across languages it matches on numbers and stray Latin letters and
|
|
190
|
+
adds noise. The answer comes from a local Ollama chat
|
|
191
|
+
model (default `qwen3:4b-instruct-2507-q4_K_M`, change it with `--llm`). It answers in the
|
|
192
|
+
language of the question and cites the passages it used.
|
|
193
|
+
|
|
194
|
+
### Docker
|
|
195
|
+
|
|
196
|
+
```bash
|
|
197
|
+
docker compose up -d ollama
|
|
198
|
+
```
|
|
199
|
+
|
|
200
|
+
```bash
|
|
201
|
+
docker compose exec ollama ollama pull bge-m3
|
|
202
|
+
```
|
|
203
|
+
|
|
204
|
+
```bash
|
|
205
|
+
docker compose run --rm app ingest /docs
|
|
206
|
+
```
|
|
207
|
+
|
|
208
|
+
## Layout
|
|
209
|
+
|
|
210
|
+
```
|
|
211
|
+
src/lanka_rag/
|
|
212
|
+
text.py Sinhala/Tamil normalization, tokenizer, char n-grams, legacy-font detector
|
|
213
|
+
models.py embedding model registry + per-model query/document prompts
|
|
214
|
+
embedders.py Ollama / Hugging Face embedders with an on-disk cache
|
|
215
|
+
bm25.py BM25 over Unicode-aware tokens or char n-grams
|
|
216
|
+
bench/ Belebele tasks, metrics, runner, report + heatmap
|
|
217
|
+
rag/ ingestion, hybrid LangChain retriever, LCEL answer chain
|
|
218
|
+
scripts/ demo corpus download
|
|
219
|
+
tests/ unit tests (no network or models needed)
|
|
220
|
+
```
|
|
221
|
+
|
|
222
|
+
## Limitations
|
|
223
|
+
|
|
224
|
+
- Belebele passages are professional translations of English Wikinews, Wikivoyage and Wikibooks
|
|
225
|
+
text (FLORES-200), not native Sri Lankan writing. Expect real gazettes and news to be harder.
|
|
226
|
+
- 488 passages is a small corpus. Absolute scores are higher than on a real document collection,
|
|
227
|
+
so read the table as a ranking of models.
|
|
228
|
+
- Many questions start with "according to the passage" (`ඡේදයට අනුව`), which adds the same noise
|
|
229
|
+
for every model.
|
|
230
|
+
- This measures retrieval only. Answer quality of the RAG chain is not benchmarked yet, and the
|
|
231
|
+
small default chat model is the weak link. Asked in English whether the Trust Fund Act lets the
|
|
232
|
+
board invest money not immediately needed, retrieval puts the exact clause at rank 2, but
|
|
233
|
+
`qwen3:4b` still replies that it could not find the answer. Same-language questions
|
|
234
|
+
(Sinhala question over Sinhala Acts) work much better. A bigger model (`--llm`) helps if you
|
|
235
|
+
have the RAM.
|
|
236
|
+
- Sinhala is expensive for LLMs: about 1.3 tokens per character in Qwen's tokenizer, so four
|
|
237
|
+
800-character chunks already make a ~4k token prompt. On an 8 GB M1 an answer takes 1–2.5
|
|
238
|
+
minutes.
|
|
239
|
+
|
|
240
|
+
## Roadmap
|
|
241
|
+
|
|
242
|
+
- [ ] FM Abhaya / DL-Manel → Unicode converter so legacy-font gazettes can be indexed
|
|
243
|
+
- [ ] Retrieval benchmark on the Sri Lankan Acts corpus with human-checked Sinhala questions
|
|
244
|
+
- [ ] Answer-quality evaluation of local LLMs (qwen3, gemma3) in Sinhala and Tamil
|
|
245
|
+
- [ ] multilingual-e5 and LaBSE results (code is in, not run yet)
|
|
246
|
+
- [ ] Small web UI
|
|
247
|
+
|
|
248
|
+
## Contributing
|
|
249
|
+
|
|
250
|
+
New models, Sinhala/Tamil test sets and bug reports are welcome. The quickest useful
|
|
251
|
+
contribution is benchmarking a model that is not in the table: see
|
|
252
|
+
[CONTRIBUTING.md](https://github.com/Nipuna-Lakruwan/lanka-rag/blob/main/CONTRIBUTING.md), or [open a model request](https://github.com/Nipuna-Lakruwan/lanka-rag/issues/new?template=model_request.yml).
|
|
253
|
+
|
|
254
|
+
## Citing
|
|
255
|
+
|
|
256
|
+
If you use the benchmark or its results, please cite this repository (GitHub's "Cite this
|
|
257
|
+
repository" button uses [CITATION.cff](https://github.com/Nipuna-Lakruwan/lanka-rag/blob/main/CITATION.cff)) and the Belebele paper.
|
|
258
|
+
|
|
259
|
+
## Credits and licenses
|
|
260
|
+
|
|
261
|
+
Code: MIT. Belebele: CC BY-SA 4.0 (Bandarkar et al., 2023), downloaded at runtime and not
|
|
262
|
+
redistributed. Sri Lankan Acts transcriptions: CC BY 4.0 (avishadilhara). Results files in
|
|
263
|
+
`results/` contain only scores and passage ids.
|