dormouse-ua 0.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dormouse_ua-0.3.0/.gitignore +17 -0
- dormouse_ua-0.3.0/LICENSE +21 -0
- dormouse_ua-0.3.0/PKG-INFO +238 -0
- dormouse_ua-0.3.0/README.md +188 -0
- dormouse_ua-0.3.0/pyproject.toml +98 -0
- dormouse_ua-0.3.0/setup.cfg +4 -0
- dormouse_ua-0.3.0/src/dormouse/__init__.py +95 -0
- dormouse_ua-0.3.0/src/dormouse/analyzer.py +86 -0
- dormouse_ua-0.3.0/src/dormouse/anthropic_proxy.py +263 -0
- dormouse_ua-0.3.0/src/dormouse/assets.py +140 -0
- dormouse_ua-0.3.0/src/dormouse/classifier.py +91 -0
- dormouse_ua-0.3.0/src/dormouse/cli.py +250 -0
- dormouse_ua-0.3.0/src/dormouse/compressor.py +138 -0
- dormouse_ua-0.3.0/src/dormouse/data/__init__.py +0 -0
- dormouse_ua-0.3.0/src/dormouse/data/replacements.json +2184 -0
- dormouse_ua-0.3.0/src/dormouse/db.py +105 -0
- dormouse_ua-0.3.0/src/dormouse/embedder.py +77 -0
- dormouse_ua-0.3.0/src/dormouse/language_guard.py +81 -0
- dormouse_ua-0.3.0/src/dormouse/lexicon_db.py +150 -0
- dormouse_ua-0.3.0/src/dormouse/local_model.py +211 -0
- dormouse_ua-0.3.0/src/dormouse/mapper.py +308 -0
- dormouse_ua-0.3.0/src/dormouse/middleware.py +368 -0
- dormouse_ua-0.3.0/src/dormouse/morphology.py +82 -0
- dormouse_ua-0.3.0/src/dormouse/optimizer.py +85 -0
- dormouse_ua-0.3.0/src/dormouse/parsers.py +239 -0
- dormouse_ua-0.3.0/src/dormouse/rule_engine.py +204 -0
- dormouse_ua-0.3.0/src/dormouse/search.py +184 -0
- dormouse_ua-0.3.0/src/dormouse/seq2seq.py +287 -0
- dormouse_ua-0.3.0/src/dormouse/stream_buffer.py +59 -0
- dormouse_ua-0.3.0/src/dormouse/style_classifier.py +82 -0
- dormouse_ua-0.3.0/src/dormouse/teapot.py +315 -0
- dormouse_ua-0.3.0/src/dormouse/unsqueeze.py +335 -0
- dormouse_ua-0.3.0/src/dormouse_ua.egg-info/PKG-INFO +238 -0
- dormouse_ua-0.3.0/src/dormouse_ua.egg-info/SOURCES.txt +62 -0
- dormouse_ua-0.3.0/src/dormouse_ua.egg-info/dependency_links.txt +1 -0
- dormouse_ua-0.3.0/src/dormouse_ua.egg-info/entry_points.txt +2 -0
- dormouse_ua-0.3.0/src/dormouse_ua.egg-info/requires.txt +44 -0
- dormouse_ua-0.3.0/src/dormouse_ua.egg-info/top_level.txt +1 -0
- dormouse_ua-0.3.0/tests/__init__.py +0 -0
- dormouse_ua-0.3.0/tests/test_analyzer.py +40 -0
- dormouse_ua-0.3.0/tests/test_anthropic_proxy.py +527 -0
- dormouse_ua-0.3.0/tests/test_brew.py +215 -0
- dormouse_ua-0.3.0/tests/test_cli.py +44 -0
- dormouse_ua-0.3.0/tests/test_cli_search.py +42 -0
- dormouse_ua-0.3.0/tests/test_compressor.py +39 -0
- dormouse_ua-0.3.0/tests/test_db.py +74 -0
- dormouse_ua-0.3.0/tests/test_embedder.py +60 -0
- dormouse_ua-0.3.0/tests/test_import.py +177 -0
- dormouse_ua-0.3.0/tests/test_language_guard.py +129 -0
- dormouse_ua-0.3.0/tests/test_local_llm.py +116 -0
- dormouse_ua-0.3.0/tests/test_local_model.py +14 -0
- dormouse_ua-0.3.0/tests/test_mapper.py +36 -0
- dormouse_ua-0.3.0/tests/test_middleware.py +332 -0
- dormouse_ua-0.3.0/tests/test_middleware_e2e.py +141 -0
- dormouse_ua-0.3.0/tests/test_openrouter.py +21 -0
- dormouse_ua-0.3.0/tests/test_optimizer.py +55 -0
- dormouse_ua-0.3.0/tests/test_parsers.py +159 -0
- dormouse_ua-0.3.0/tests/test_rule_engine.py +105 -0
- dormouse_ua-0.3.0/tests/test_search.py +97 -0
- dormouse_ua-0.3.0/tests/test_seq2seq.py +76 -0
- dormouse_ua-0.3.0/tests/test_stream_buffer.py +88 -0
- dormouse_ua-0.3.0/tests/test_style_classifier.py +71 -0
- dormouse_ua-0.3.0/tests/test_teapot.py +248 -0
- dormouse_ua-0.3.0/tests/test_unsqueeze.py +139 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Daria Chuprina
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,238 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: dormouse-ua
|
|
3
|
+
Version: 0.3.0
|
|
4
|
+
Summary: Оптимізація українських текстів для LLM: менше токенів, краще розуміння
|
|
5
|
+
Author-email: Daria Chuprina <daria@lazysoft.dev>
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/ChuprinaDaria/dormouse
|
|
8
|
+
Keywords: ukrainian,nlp,llm,tokenization,optimization
|
|
9
|
+
Classifier: Development Status :: 3 - Alpha
|
|
10
|
+
Classifier: Intended Audience :: Developers
|
|
11
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
12
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
13
|
+
Classifier: Topic :: Text Processing :: Linguistic
|
|
14
|
+
Requires-Python: >=3.11
|
|
15
|
+
Description-Content-Type: text/markdown
|
|
16
|
+
License-File: LICENSE
|
|
17
|
+
Requires-Dist: click>=8.1
|
|
18
|
+
Provides-Extra: morph
|
|
19
|
+
Requires-Dist: pymorphy3>=2.0; extra == "morph"
|
|
20
|
+
Provides-Extra: tokens
|
|
21
|
+
Requires-Dist: tiktoken>=0.7; extra == "tokens"
|
|
22
|
+
Requires-Dist: anthropic>=0.40; extra == "tokens"
|
|
23
|
+
Provides-Extra: embeddings
|
|
24
|
+
Requires-Dist: sentence-transformers>=3.0; extra == "embeddings"
|
|
25
|
+
Provides-Extra: compress
|
|
26
|
+
Requires-Dist: llmlingua>=0.2; extra == "compress"
|
|
27
|
+
Provides-Extra: translate
|
|
28
|
+
Requires-Dist: transformers>=4.40; extra == "translate"
|
|
29
|
+
Requires-Dist: sentencepiece>=0.2; extra == "translate"
|
|
30
|
+
Provides-Extra: ml
|
|
31
|
+
Requires-Dist: torch>=2.0; extra == "ml"
|
|
32
|
+
Provides-Extra: style
|
|
33
|
+
Requires-Dist: scikit-learn>=1.3; extra == "style"
|
|
34
|
+
Requires-Dist: numpy>=1.24; extra == "style"
|
|
35
|
+
Provides-Extra: search
|
|
36
|
+
Requires-Dist: sentence-transformers>=3.0; extra == "search"
|
|
37
|
+
Requires-Dist: openpyxl>=3.1; extra == "search"
|
|
38
|
+
Requires-Dist: pymupdf>=1.24; extra == "search"
|
|
39
|
+
Provides-Extra: middleware
|
|
40
|
+
Requires-Dist: openai>=1.0; extra == "middleware"
|
|
41
|
+
Provides-Extra: anthropic-middleware
|
|
42
|
+
Requires-Dist: anthropic>=0.40; extra == "anthropic-middleware"
|
|
43
|
+
Provides-Extra: all
|
|
44
|
+
Requires-Dist: dormouse[anthropic-middleware,compress,embeddings,middleware,ml,morph,search,style,tokens,translate]; extra == "all"
|
|
45
|
+
Provides-Extra: dev
|
|
46
|
+
Requires-Dist: pytest>=8.0; extra == "dev"
|
|
47
|
+
Requires-Dist: ruff>=0.5; extra == "dev"
|
|
48
|
+
Requires-Dist: pymorphy3>=2.0; extra == "dev"
|
|
49
|
+
Dynamic: license-file
|
|
50
|
+
|
|
51
|
+
# dormouse
|
|
52
|
+
|
|
53
|
+
**Ukrainian text optimizer for LLMs** — fewer tokens, better comprehension.
|
|
54
|
+
|
|
55
|
+
Normalizes surzhyk, slang, fillers, and maps to English for cloud LLMs. Saves 60-73% tokens while *improving* response quality.
|
|
56
|
+
|
|
57
|
+
> **UA:** Оптимізація українських текстів для LLM. Нормалізує суржик, сленг, мат — і стискає в англійську для Claude/GPT. Економія 60-73% токенів, якість відповідей зростає зі 67% до 100%.
|
|
58
|
+
|
|
59
|
+
## Results
|
|
60
|
+
|
|
61
|
+
Tested on 53,351 texts (Telegram corpus + books), 12 IT prompts across 4 GPT models:
|
|
62
|
+
|
|
63
|
+
| Metric | Value |
|
|
64
|
+
|--------|-------|
|
|
65
|
+
| Token savings (cloud) | **73%** |
|
|
66
|
+
| Token savings (without seq2seq) | **49%** |
|
|
67
|
+
| Lexicon coverage | **88%** |
|
|
68
|
+
| Seq2seq exact match | **98.2%** |
|
|
69
|
+
| GPT response quality (original UA) | 67% |
|
|
70
|
+
| GPT response quality (squeezed EN) | **100%** |
|
|
71
|
+
| Quality preservation | **150%** (squeezed > original) |
|
|
72
|
+
|
|
73
|
+
```
|
|
74
|
+
Original UA: "блін продакшн впав після деплою, що робити першим"
|
|
75
|
+
Squeezed EN: "damn production crashed after deploy, what do first"
|
|
76
|
+
Tokens: 45 → 12 (-73%)
|
|
77
|
+
GPT accuracy: 67% → 100%
|
|
78
|
+
```
|
|
79
|
+
|
|
80
|
+
## How it works
|
|
81
|
+
|
|
82
|
+
```mermaid
|
|
83
|
+
graph LR
|
|
84
|
+
A[UA text<br/>surzhyk, slang] --> B[crack_open<br/>normalize]
|
|
85
|
+
B --> C[compress<br/>remove fillers]
|
|
86
|
+
C --> D[map_to_en<br/>lexicon + seq2seq]
|
|
87
|
+
D --> E[EN compressed<br/>for LLM]
|
|
88
|
+
|
|
89
|
+
style A fill:#fdd,stroke:#c33
|
|
90
|
+
style E fill:#dfd,stroke:#3a3
|
|
91
|
+
```
|
|
92
|
+
|
|
93
|
+
| Layer | What it does | How |
|
|
94
|
+
|-------|-------------|-----|
|
|
95
|
+
| **crack_open** | surzhyk, slang, profanity → standard UA | 360 rules + pymorphy3 lemmatization |
|
|
96
|
+
| **compress** | remove fillers, intensifiers, noise | rule-based pattern matching |
|
|
97
|
+
| **map_to_en** | UA → compact English | 47K lexicon + seq2seq (28K expression pairs) |
|
|
98
|
+
|
|
99
|
+
## Install
|
|
100
|
+
|
|
101
|
+
```bash
|
|
102
|
+
pip install dormouse
|
|
103
|
+
|
|
104
|
+
# With morphological analysis (recommended)
|
|
105
|
+
pip install dormouse[morph]
|
|
106
|
+
|
|
107
|
+
# Everything
|
|
108
|
+
pip install dormouse[all]
|
|
109
|
+
```
|
|
110
|
+
|
|
111
|
+
## Quick start
|
|
112
|
+
|
|
113
|
+
```python
|
|
114
|
+
from dormouse import squeeze
|
|
115
|
+
|
|
116
|
+
# Normalize only (layers 1+2)
|
|
117
|
+
squeeze("шо там по баґу, пофікси плз")
|
|
118
|
+
# → "що там по помилці, виправ"
|
|
119
|
+
|
|
120
|
+
# Cloud mode — compress for Claude/GPT (layers 1+2+3)
|
|
121
|
+
squeeze("ваще нормально, канєшно зробимо", target="cloud")
|
|
122
|
+
# → "generally ok, sure do"
|
|
123
|
+
```
|
|
124
|
+
|
|
125
|
+
### SDK Middleware (drop-in)
|
|
126
|
+
|
|
127
|
+
```python
|
|
128
|
+
from openai import OpenAI
|
|
129
|
+
from dormouse import DormouseClient
|
|
130
|
+
|
|
131
|
+
client = DormouseClient(OpenAI()) # or Anthropic()
|
|
132
|
+
|
|
133
|
+
response = client.chat.completions.create(
|
|
134
|
+
model="gpt-4o-mini",
|
|
135
|
+
messages=[{"role": "user", "content": "шо там по деплою, він ваще не робе"}],
|
|
136
|
+
)
|
|
137
|
+
# Prompt: squeeze → EN → GPT → unsqueeze → Ukrainian response
|
|
138
|
+
```
|
|
139
|
+
|
|
140
|
+
### Semantic search
|
|
141
|
+
|
|
142
|
+
```python
|
|
143
|
+
from dormouse import stir, mumble, sip
|
|
144
|
+
|
|
145
|
+
stir("report.pdf") # index
|
|
146
|
+
results = mumble("холодні закуски") # search by meaning
|
|
147
|
+
topics = sip("data.xlsx", topics=["HR", "finance"]) # classify
|
|
148
|
+
```
|
|
149
|
+
|
|
150
|
+
### CLI
|
|
151
|
+
|
|
152
|
+
```bash
|
|
153
|
+
dormouse squeeze "шо там по баґу" -t cloud
|
|
154
|
+
dormouse stir book.pdf
|
|
155
|
+
dormouse mumble "головний герой"
|
|
156
|
+
```
|
|
157
|
+
|
|
158
|
+
## Comparison with alternatives
|
|
159
|
+
|
|
160
|
+
| Tool | Ukrainian | Token savings | Approach | Quality impact |
|
|
161
|
+
|------|:---------:|:------------:|----------|:--------------:|
|
|
162
|
+
| **dormouse** | native | **73%** | normalize + compress + translate | **+50% quality** |
|
|
163
|
+
| [LLMLingua](https://github.com/microsoft/LLMLingua) | no | up to 20x | ML perplexity pruning (GPT-2/LLaMA) | -5-15% |
|
|
164
|
+
| [Selective Context](https://github.com/liyucheng09/Selective_Context) | no | 40-50% | self-information filtering | -10-20% |
|
|
165
|
+
| [token-reducer](https://pypi.org/project/token-reducer/) | no | 50-75% | 6-stage pipeline, AST for code | neutral |
|
|
166
|
+
| [shrink-prompt](https://pypi.org/project/shrink-prompt/) | no | 30-70% | domain-specific rules (<20ms) | neutral |
|
|
167
|
+
| Google Translate → EN | partial | 30-40% | full translation | variable |
|
|
168
|
+
|
|
169
|
+
**Why dormouse is different:**
|
|
170
|
+
|
|
171
|
+
The problem: Ukrainian Cyrillic costs [3-4x more tokens](https://www.frontiersin.org/journals/artificial-intelligence/articles/10.3389/frai.2025.1538165/full) than equivalent English text in GPT-4/Claude.
|
|
172
|
+
|
|
173
|
+
All existing tools (LLMLingua, Selective Context, token-reducer) compress *already English* text by removing information. dormouse solves the problem one level earlier — transforms expensive Ukrainian (3-4 tokens/word) into cheap English (1-1.5 tokens/word) while **preserving all meaning**.
|
|
174
|
+
|
|
175
|
+
No other tool specifically optimizes Ukrainian for LLMs.
|
|
176
|
+
|
|
177
|
+
## Use cases
|
|
178
|
+
|
|
179
|
+
**Cost reduction** — Ukrainian Cyrillic encodes into 2-4x more tokens than equivalent English. dormouse saves 60-73% on input tokens.
|
|
180
|
+
|
|
181
|
+
**Chatbots & support** — Users write in surzhyk/slang, dormouse normalizes before LLM, GPT gives concrete answers instead of generic responses.
|
|
182
|
+
|
|
183
|
+
**RAG & document search** — User searches in slang, documents are in literary language. dormouse normalizes both sides → finds by meaning.
|
|
184
|
+
|
|
185
|
+
**AI agents** — Long chains of actions eat context window. 73% compression = 73% more "memory" for the agent.
|
|
186
|
+
|
|
187
|
+
**Batch processing** — 10K comments through GPT for sentiment analysis. Squeeze first → cheaper and faster.
|
|
188
|
+
|
|
189
|
+
## Eval details
|
|
190
|
+
|
|
191
|
+
Full evaluation ran for 4 days on 53,351 texts:
|
|
192
|
+
|
|
193
|
+
```
|
|
194
|
+
Corpus: 53,351 texts (Telegram + books)
|
|
195
|
+
Squeeze speed: 606 texts/sec (normalization)
|
|
196
|
+
Seq2seq model: 7.3M params, 28K expression pairs
|
|
197
|
+
Stir/mumble: 8,441 chunks indexed, search ~600ms
|
|
198
|
+
Sip classification: 99% texts classified (8 topics)
|
|
199
|
+
```
|
|
200
|
+
|
|
201
|
+
### HF Inference API (small models)
|
|
202
|
+
|
|
203
|
+
| Model | UA score | Squeezed EN score | Delta |
|
|
204
|
+
|-------|:--------:|:-----------------:|:-----:|
|
|
205
|
+
| Qwen2.5-72B | 4.9/5 | 4.5/5 | -0.4 |
|
|
206
|
+
| Qwen2.5-7B | 4.4/5 | 3.6/5 | -0.8 |
|
|
207
|
+
| Llama-3.2-1B | 2.7/5 | 2.8/5 | +0.1 |
|
|
208
|
+
|
|
209
|
+
> Squeeze works great for cloud models (GPT, Claude, 70B+). For small models (<7B), use `brew()` with native Ukrainian — they understand UA better than squeezed EN.
|
|
210
|
+
|
|
211
|
+
## Architecture
|
|
212
|
+
|
|
213
|
+
```
|
|
214
|
+
src/dormouse/
|
|
215
|
+
├── optimizer.py — squeeze() main pipeline
|
|
216
|
+
├── rule_engine.py — normalization (360 rules + pymorphy3)
|
|
217
|
+
├── compressor.py — filler/noise removal
|
|
218
|
+
├── mapper.py — UA→EN via lexicon + lemma + transliteration
|
|
219
|
+
├── seq2seq.py — expression translator (GRU encoder-decoder)
|
|
220
|
+
├── teapot.py — stir/mumble/sip/brew (search + LLM)
|
|
221
|
+
├── embedder.py — sentence-transformers wrapper
|
|
222
|
+
├── middleware.py — OpenAI/Anthropic SDK proxy
|
|
223
|
+
├── cli.py — Click CLI
|
|
224
|
+
└── assets.py — lazy download of models/data
|
|
225
|
+
```
|
|
226
|
+
|
|
227
|
+
## Development
|
|
228
|
+
|
|
229
|
+
```bash
|
|
230
|
+
git clone https://github.com/ChuprinaDaria/dormouse
|
|
231
|
+
cd dormouse
|
|
232
|
+
pip install -e ".[dev,morph]"
|
|
233
|
+
DORMOUSE_DATA_DIR=./data pytest tests/ -v
|
|
234
|
+
```
|
|
235
|
+
|
|
236
|
+
## License
|
|
237
|
+
|
|
238
|
+
MIT
|
|
@@ -0,0 +1,188 @@
|
|
|
1
|
+
# dormouse
|
|
2
|
+
|
|
3
|
+
**Ukrainian text optimizer for LLMs** — fewer tokens, better comprehension.
|
|
4
|
+
|
|
5
|
+
Normalizes surzhyk, slang, fillers, and maps to English for cloud LLMs. Saves 60-73% tokens while *improving* response quality.
|
|
6
|
+
|
|
7
|
+
> **UA:** Оптимізація українських текстів для LLM. Нормалізує суржик, сленг, мат — і стискає в англійську для Claude/GPT. Економія 60-73% токенів, якість відповідей зростає зі 67% до 100%.
|
|
8
|
+
|
|
9
|
+
## Results
|
|
10
|
+
|
|
11
|
+
Tested on 53,351 texts (Telegram corpus + books), 12 IT prompts across 4 GPT models:
|
|
12
|
+
|
|
13
|
+
| Metric | Value |
|
|
14
|
+
|--------|-------|
|
|
15
|
+
| Token savings (cloud) | **73%** |
|
|
16
|
+
| Token savings (without seq2seq) | **49%** |
|
|
17
|
+
| Lexicon coverage | **88%** |
|
|
18
|
+
| Seq2seq exact match | **98.2%** |
|
|
19
|
+
| GPT response quality (original UA) | 67% |
|
|
20
|
+
| GPT response quality (squeezed EN) | **100%** |
|
|
21
|
+
| Quality preservation | **150%** (squeezed > original) |
|
|
22
|
+
|
|
23
|
+
```
|
|
24
|
+
Original UA: "блін продакшн впав після деплою, що робити першим"
|
|
25
|
+
Squeezed EN: "damn production crashed after deploy, what do first"
|
|
26
|
+
Tokens: 45 → 12 (-73%)
|
|
27
|
+
GPT accuracy: 67% → 100%
|
|
28
|
+
```
|
|
29
|
+
|
|
30
|
+
## How it works
|
|
31
|
+
|
|
32
|
+
```mermaid
|
|
33
|
+
graph LR
|
|
34
|
+
A[UA text<br/>surzhyk, slang] --> B[crack_open<br/>normalize]
|
|
35
|
+
B --> C[compress<br/>remove fillers]
|
|
36
|
+
C --> D[map_to_en<br/>lexicon + seq2seq]
|
|
37
|
+
D --> E[EN compressed<br/>for LLM]
|
|
38
|
+
|
|
39
|
+
style A fill:#fdd,stroke:#c33
|
|
40
|
+
style E fill:#dfd,stroke:#3a3
|
|
41
|
+
```
|
|
42
|
+
|
|
43
|
+
| Layer | What it does | How |
|
|
44
|
+
|-------|-------------|-----|
|
|
45
|
+
| **crack_open** | surzhyk, slang, profanity → standard UA | 360 rules + pymorphy3 lemmatization |
|
|
46
|
+
| **compress** | remove fillers, intensifiers, noise | rule-based pattern matching |
|
|
47
|
+
| **map_to_en** | UA → compact English | 47K lexicon + seq2seq (28K expression pairs) |
|
|
48
|
+
|
|
49
|
+
## Install
|
|
50
|
+
|
|
51
|
+
```bash
|
|
52
|
+
pip install dormouse
|
|
53
|
+
|
|
54
|
+
# With morphological analysis (recommended)
|
|
55
|
+
pip install dormouse[morph]
|
|
56
|
+
|
|
57
|
+
# Everything
|
|
58
|
+
pip install dormouse[all]
|
|
59
|
+
```
|
|
60
|
+
|
|
61
|
+
## Quick start
|
|
62
|
+
|
|
63
|
+
```python
|
|
64
|
+
from dormouse import squeeze
|
|
65
|
+
|
|
66
|
+
# Normalize only (layers 1+2)
|
|
67
|
+
squeeze("шо там по баґу, пофікси плз")
|
|
68
|
+
# → "що там по помилці, виправ"
|
|
69
|
+
|
|
70
|
+
# Cloud mode — compress for Claude/GPT (layers 1+2+3)
|
|
71
|
+
squeeze("ваще нормально, канєшно зробимо", target="cloud")
|
|
72
|
+
# → "generally ok, sure do"
|
|
73
|
+
```
|
|
74
|
+
|
|
75
|
+
### SDK Middleware (drop-in)
|
|
76
|
+
|
|
77
|
+
```python
|
|
78
|
+
from openai import OpenAI
|
|
79
|
+
from dormouse import DormouseClient
|
|
80
|
+
|
|
81
|
+
client = DormouseClient(OpenAI()) # or Anthropic()
|
|
82
|
+
|
|
83
|
+
response = client.chat.completions.create(
|
|
84
|
+
model="gpt-4o-mini",
|
|
85
|
+
messages=[{"role": "user", "content": "шо там по деплою, він ваще не робе"}],
|
|
86
|
+
)
|
|
87
|
+
# Prompt: squeeze → EN → GPT → unsqueeze → Ukrainian response
|
|
88
|
+
```
|
|
89
|
+
|
|
90
|
+
### Semantic search
|
|
91
|
+
|
|
92
|
+
```python
|
|
93
|
+
from dormouse import stir, mumble, sip
|
|
94
|
+
|
|
95
|
+
stir("report.pdf") # index
|
|
96
|
+
results = mumble("холодні закуски") # search by meaning
|
|
97
|
+
topics = sip("data.xlsx", topics=["HR", "finance"]) # classify
|
|
98
|
+
```
|
|
99
|
+
|
|
100
|
+
### CLI
|
|
101
|
+
|
|
102
|
+
```bash
|
|
103
|
+
dormouse squeeze "шо там по баґу" -t cloud
|
|
104
|
+
dormouse stir book.pdf
|
|
105
|
+
dormouse mumble "головний герой"
|
|
106
|
+
```
|
|
107
|
+
|
|
108
|
+
## Comparison with alternatives
|
|
109
|
+
|
|
110
|
+
| Tool | Ukrainian | Token savings | Approach | Quality impact |
|
|
111
|
+
|------|:---------:|:------------:|----------|:--------------:|
|
|
112
|
+
| **dormouse** | native | **73%** | normalize + compress + translate | **+50% quality** |
|
|
113
|
+
| [LLMLingua](https://github.com/microsoft/LLMLingua) | no | up to 20x | ML perplexity pruning (GPT-2/LLaMA) | -5-15% |
|
|
114
|
+
| [Selective Context](https://github.com/liyucheng09/Selective_Context) | no | 40-50% | self-information filtering | -10-20% |
|
|
115
|
+
| [token-reducer](https://pypi.org/project/token-reducer/) | no | 50-75% | 6-stage pipeline, AST for code | neutral |
|
|
116
|
+
| [shrink-prompt](https://pypi.org/project/shrink-prompt/) | no | 30-70% | domain-specific rules (<20ms) | neutral |
|
|
117
|
+
| Google Translate → EN | partial | 30-40% | full translation | variable |
|
|
118
|
+
|
|
119
|
+
**Why dormouse is different:**
|
|
120
|
+
|
|
121
|
+
The problem: Ukrainian Cyrillic costs [3-4x more tokens](https://www.frontiersin.org/journals/artificial-intelligence/articles/10.3389/frai.2025.1538165/full) than equivalent English text in GPT-4/Claude.
|
|
122
|
+
|
|
123
|
+
All existing tools (LLMLingua, Selective Context, token-reducer) compress *already English* text by removing information. dormouse solves the problem one level earlier — transforms expensive Ukrainian (3-4 tokens/word) into cheap English (1-1.5 tokens/word) while **preserving all meaning**.
|
|
124
|
+
|
|
125
|
+
No other tool specifically optimizes Ukrainian for LLMs.
|
|
126
|
+
|
|
127
|
+
## Use cases
|
|
128
|
+
|
|
129
|
+
**Cost reduction** — Ukrainian Cyrillic encodes into 2-4x more tokens than equivalent English. dormouse saves 60-73% on input tokens.
|
|
130
|
+
|
|
131
|
+
**Chatbots & support** — Users write in surzhyk/slang, dormouse normalizes before LLM, GPT gives concrete answers instead of generic responses.
|
|
132
|
+
|
|
133
|
+
**RAG & document search** — User searches in slang, documents are in literary language. dormouse normalizes both sides → finds by meaning.
|
|
134
|
+
|
|
135
|
+
**AI agents** — Long chains of actions eat context window. 73% compression = 73% more "memory" for the agent.
|
|
136
|
+
|
|
137
|
+
**Batch processing** — 10K comments through GPT for sentiment analysis. Squeeze first → cheaper and faster.
|
|
138
|
+
|
|
139
|
+
## Eval details
|
|
140
|
+
|
|
141
|
+
Full evaluation ran for 4 days on 53,351 texts:
|
|
142
|
+
|
|
143
|
+
```
|
|
144
|
+
Corpus: 53,351 texts (Telegram + books)
|
|
145
|
+
Squeeze speed: 606 texts/sec (normalization)
|
|
146
|
+
Seq2seq model: 7.3M params, 28K expression pairs
|
|
147
|
+
Stir/mumble: 8,441 chunks indexed, search ~600ms
|
|
148
|
+
Sip classification: 99% texts classified (8 topics)
|
|
149
|
+
```
|
|
150
|
+
|
|
151
|
+
### HF Inference API (small models)
|
|
152
|
+
|
|
153
|
+
| Model | UA score | Squeezed EN score | Delta |
|
|
154
|
+
|-------|:--------:|:-----------------:|:-----:|
|
|
155
|
+
| Qwen2.5-72B | 4.9/5 | 4.5/5 | -0.4 |
|
|
156
|
+
| Qwen2.5-7B | 4.4/5 | 3.6/5 | -0.8 |
|
|
157
|
+
| Llama-3.2-1B | 2.7/5 | 2.8/5 | +0.1 |
|
|
158
|
+
|
|
159
|
+
> Squeeze works great for cloud models (GPT, Claude, 70B+). For small models (<7B), use `brew()` with native Ukrainian — they understand UA better than squeezed EN.
|
|
160
|
+
|
|
161
|
+
## Architecture
|
|
162
|
+
|
|
163
|
+
```
|
|
164
|
+
src/dormouse/
|
|
165
|
+
├── optimizer.py — squeeze() main pipeline
|
|
166
|
+
├── rule_engine.py — normalization (360 rules + pymorphy3)
|
|
167
|
+
├── compressor.py — filler/noise removal
|
|
168
|
+
├── mapper.py — UA→EN via lexicon + lemma + transliteration
|
|
169
|
+
├── seq2seq.py — expression translator (GRU encoder-decoder)
|
|
170
|
+
├── teapot.py — stir/mumble/sip/brew (search + LLM)
|
|
171
|
+
├── embedder.py — sentence-transformers wrapper
|
|
172
|
+
├── middleware.py — OpenAI/Anthropic SDK proxy
|
|
173
|
+
├── cli.py — Click CLI
|
|
174
|
+
└── assets.py — lazy download of models/data
|
|
175
|
+
```
|
|
176
|
+
|
|
177
|
+
## Development
|
|
178
|
+
|
|
179
|
+
```bash
|
|
180
|
+
git clone https://github.com/ChuprinaDaria/dormouse
|
|
181
|
+
cd dormouse
|
|
182
|
+
pip install -e ".[dev,morph]"
|
|
183
|
+
DORMOUSE_DATA_DIR=./data pytest tests/ -v
|
|
184
|
+
```
|
|
185
|
+
|
|
186
|
+
## License
|
|
187
|
+
|
|
188
|
+
MIT
|
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=68.0", "setuptools-scm>=8.0"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "dormouse-ua"
|
|
7
|
+
version = "0.3.0"
|
|
8
|
+
description = "Оптимізація українських текстів для LLM: менше токенів, краще розуміння"
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
license = "MIT"
|
|
11
|
+
requires-python = ">=3.11"
|
|
12
|
+
authors = [
|
|
13
|
+
{name = "Daria Chuprina", email = "daria@lazysoft.dev"},
|
|
14
|
+
]
|
|
15
|
+
keywords = ["ukrainian", "nlp", "llm", "tokenization", "optimization"]
|
|
16
|
+
classifiers = [
|
|
17
|
+
"Development Status :: 3 - Alpha",
|
|
18
|
+
"Intended Audience :: Developers",
|
|
19
|
+
"Programming Language :: Python :: 3.11",
|
|
20
|
+
"Programming Language :: Python :: 3.12",
|
|
21
|
+
"Topic :: Text Processing :: Linguistic",
|
|
22
|
+
]
|
|
23
|
+
dependencies = [
|
|
24
|
+
"click>=8.1",
|
|
25
|
+
]
|
|
26
|
+
|
|
27
|
+
[project.optional-dependencies]
|
|
28
|
+
morph = [
|
|
29
|
+
"pymorphy3>=2.0",
|
|
30
|
+
]
|
|
31
|
+
tokens = [
|
|
32
|
+
"tiktoken>=0.7",
|
|
33
|
+
"anthropic>=0.40",
|
|
34
|
+
]
|
|
35
|
+
embeddings = [
|
|
36
|
+
"sentence-transformers>=3.0",
|
|
37
|
+
]
|
|
38
|
+
compress = [
|
|
39
|
+
"llmlingua>=0.2",
|
|
40
|
+
]
|
|
41
|
+
translate = [
|
|
42
|
+
"transformers>=4.40",
|
|
43
|
+
"sentencepiece>=0.2",
|
|
44
|
+
]
|
|
45
|
+
ml = [
|
|
46
|
+
"torch>=2.0",
|
|
47
|
+
]
|
|
48
|
+
style = [
|
|
49
|
+
"scikit-learn>=1.3",
|
|
50
|
+
"numpy>=1.24",
|
|
51
|
+
]
|
|
52
|
+
search = [
|
|
53
|
+
"sentence-transformers>=3.0",
|
|
54
|
+
"openpyxl>=3.1",
|
|
55
|
+
"pymupdf>=1.24",
|
|
56
|
+
]
|
|
57
|
+
middleware = [
|
|
58
|
+
"openai>=1.0",
|
|
59
|
+
]
|
|
60
|
+
anthropic-middleware = [
|
|
61
|
+
"anthropic>=0.40",
|
|
62
|
+
]
|
|
63
|
+
all = [
|
|
64
|
+
"dormouse[morph,tokens,embeddings,compress,translate,ml,style,search,middleware,anthropic-middleware]",
|
|
65
|
+
]
|
|
66
|
+
dev = [
|
|
67
|
+
"pytest>=8.0",
|
|
68
|
+
"ruff>=0.5",
|
|
69
|
+
"pymorphy3>=2.0",
|
|
70
|
+
]
|
|
71
|
+
|
|
72
|
+
[project.scripts]
|
|
73
|
+
dormouse = "dormouse.cli:main"
|
|
74
|
+
|
|
75
|
+
[project.urls]
|
|
76
|
+
Homepage = "https://github.com/ChuprinaDaria/dormouse"
|
|
77
|
+
|
|
78
|
+
[tool.setuptools.packages.find]
|
|
79
|
+
where = ["src"]
|
|
80
|
+
|
|
81
|
+
[tool.setuptools.package-data]
|
|
82
|
+
dormouse = ["data/*.json"]
|
|
83
|
+
|
|
84
|
+
[tool.ruff]
|
|
85
|
+
target-version = "py311"
|
|
86
|
+
line-length = 99
|
|
87
|
+
|
|
88
|
+
[tool.ruff.lint]
|
|
89
|
+
select = ["E", "F", "I", "W"]
|
|
90
|
+
|
|
91
|
+
[tool.ruff.lint.per-file-ignores]
|
|
92
|
+
"tests/*" = ["E402"]
|
|
93
|
+
|
|
94
|
+
[tool.pytest.ini_options]
|
|
95
|
+
testpaths = ["tests"]
|
|
96
|
+
markers = [
|
|
97
|
+
"e2e: end-to-end tests with real API (require API keys)",
|
|
98
|
+
]
|
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
"""dormouse — оптимізація українських текстів для LLM."""
|
|
2
|
+
|
|
3
|
+
__version__ = "0.3.0"
|
|
4
|
+
|
|
5
|
+
from dormouse.optimizer import SqueezedText, squeeze, squeeze_batch
|
|
6
|
+
|
|
7
|
+
__all__ = [
|
|
8
|
+
"squeeze",
|
|
9
|
+
"squeeze_batch",
|
|
10
|
+
"SqueezedText",
|
|
11
|
+
"__version__",
|
|
12
|
+
]
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def nibble(text: str):
|
|
16
|
+
"""Бенчмарк токенізації (потребує pip install dormouse[tokens])."""
|
|
17
|
+
from dormouse.analyzer import nibble as _nibble
|
|
18
|
+
|
|
19
|
+
return _nibble(text)
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def nibble_batch(texts: list[str]):
|
|
23
|
+
"""Батч-аналіз токенізації."""
|
|
24
|
+
from dormouse.analyzer import nibble_batch as _nibble_batch
|
|
25
|
+
|
|
26
|
+
return _nibble_batch(texts)
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def sniff(
|
|
30
|
+
items: list[str],
|
|
31
|
+
categories: list[str],
|
|
32
|
+
squeeze_first: bool = False,
|
|
33
|
+
):
|
|
34
|
+
"""Класифікація через embeddings (потребує pip install dormouse[embeddings])."""
|
|
35
|
+
from dormouse.classifier import sniff as _sniff
|
|
36
|
+
|
|
37
|
+
return _sniff(items, categories, squeeze_first=squeeze_first)
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def DormouseClient(client, **kwargs):
|
|
41
|
+
"""SDK wrapper для OpenAI (потребує pip install dormouse[middleware])."""
|
|
42
|
+
from dormouse.middleware import DormouseClient as _DormouseClient
|
|
43
|
+
|
|
44
|
+
return _DormouseClient(client, **kwargs)
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def stir(source, **kwargs):
|
|
48
|
+
"""Завантажити і проіндексувати файл (потребує pip install dormouse[search])."""
|
|
49
|
+
from dormouse.teapot import Teapot
|
|
50
|
+
|
|
51
|
+
try:
|
|
52
|
+
from dormouse.embedder import Embedder
|
|
53
|
+
|
|
54
|
+
emb = Embedder()
|
|
55
|
+
except ImportError:
|
|
56
|
+
emb = None
|
|
57
|
+
t = Teapot(embedder=emb)
|
|
58
|
+
return t.stir(source, **kwargs)
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def mumble(query, index=None, top_k=10):
|
|
62
|
+
"""Семантичний пошук по проіндексованих даних."""
|
|
63
|
+
from dormouse.teapot import Teapot
|
|
64
|
+
|
|
65
|
+
if index is not None:
|
|
66
|
+
return index.mumble(query, top_k=top_k)
|
|
67
|
+
t = Teapot()
|
|
68
|
+
return t.mumble(query, top_k=top_k)
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def sip(source, topics, threshold=0.5, **kwargs):
|
|
72
|
+
"""Класифікація тексту по заданих темах."""
|
|
73
|
+
from dormouse.embedder import Embedder
|
|
74
|
+
from dormouse.teapot import Teapot
|
|
75
|
+
|
|
76
|
+
emb = Embedder()
|
|
77
|
+
t = Teapot(embedder=emb)
|
|
78
|
+
return t.sip(source, topics, threshold=threshold)
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def brew(query, *, llm, index=None, extract=None, strategy="hierarchical", limit=50):
|
|
82
|
+
"""LLM-powered пошук і витягування (потребує pip install dormouse[search])."""
|
|
83
|
+
from dormouse.teapot import Teapot
|
|
84
|
+
|
|
85
|
+
if index is not None:
|
|
86
|
+
return index.brew(query, llm=llm, extract=extract, strategy=strategy, limit=limit)
|
|
87
|
+
t = Teapot()
|
|
88
|
+
return t.brew(query, llm=llm, extract=extract, strategy=strategy, limit=limit)
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def LocalLLM(model="MamayLM-4B", **kwargs):
|
|
92
|
+
"""Уніфікований інтерфейс до локальної LLM."""
|
|
93
|
+
from dormouse.local_model import LocalLLM as _LocalLLM
|
|
94
|
+
|
|
95
|
+
return _LocalLLM(model, **kwargs)
|