dormouse-ua 0.3.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (64) hide show
  1. dormouse_ua-0.3.0/.gitignore +17 -0
  2. dormouse_ua-0.3.0/LICENSE +21 -0
  3. dormouse_ua-0.3.0/PKG-INFO +238 -0
  4. dormouse_ua-0.3.0/README.md +188 -0
  5. dormouse_ua-0.3.0/pyproject.toml +98 -0
  6. dormouse_ua-0.3.0/setup.cfg +4 -0
  7. dormouse_ua-0.3.0/src/dormouse/__init__.py +95 -0
  8. dormouse_ua-0.3.0/src/dormouse/analyzer.py +86 -0
  9. dormouse_ua-0.3.0/src/dormouse/anthropic_proxy.py +263 -0
  10. dormouse_ua-0.3.0/src/dormouse/assets.py +140 -0
  11. dormouse_ua-0.3.0/src/dormouse/classifier.py +91 -0
  12. dormouse_ua-0.3.0/src/dormouse/cli.py +250 -0
  13. dormouse_ua-0.3.0/src/dormouse/compressor.py +138 -0
  14. dormouse_ua-0.3.0/src/dormouse/data/__init__.py +0 -0
  15. dormouse_ua-0.3.0/src/dormouse/data/replacements.json +2184 -0
  16. dormouse_ua-0.3.0/src/dormouse/db.py +105 -0
  17. dormouse_ua-0.3.0/src/dormouse/embedder.py +77 -0
  18. dormouse_ua-0.3.0/src/dormouse/language_guard.py +81 -0
  19. dormouse_ua-0.3.0/src/dormouse/lexicon_db.py +150 -0
  20. dormouse_ua-0.3.0/src/dormouse/local_model.py +211 -0
  21. dormouse_ua-0.3.0/src/dormouse/mapper.py +308 -0
  22. dormouse_ua-0.3.0/src/dormouse/middleware.py +368 -0
  23. dormouse_ua-0.3.0/src/dormouse/morphology.py +82 -0
  24. dormouse_ua-0.3.0/src/dormouse/optimizer.py +85 -0
  25. dormouse_ua-0.3.0/src/dormouse/parsers.py +239 -0
  26. dormouse_ua-0.3.0/src/dormouse/rule_engine.py +204 -0
  27. dormouse_ua-0.3.0/src/dormouse/search.py +184 -0
  28. dormouse_ua-0.3.0/src/dormouse/seq2seq.py +287 -0
  29. dormouse_ua-0.3.0/src/dormouse/stream_buffer.py +59 -0
  30. dormouse_ua-0.3.0/src/dormouse/style_classifier.py +82 -0
  31. dormouse_ua-0.3.0/src/dormouse/teapot.py +315 -0
  32. dormouse_ua-0.3.0/src/dormouse/unsqueeze.py +335 -0
  33. dormouse_ua-0.3.0/src/dormouse_ua.egg-info/PKG-INFO +238 -0
  34. dormouse_ua-0.3.0/src/dormouse_ua.egg-info/SOURCES.txt +62 -0
  35. dormouse_ua-0.3.0/src/dormouse_ua.egg-info/dependency_links.txt +1 -0
  36. dormouse_ua-0.3.0/src/dormouse_ua.egg-info/entry_points.txt +2 -0
  37. dormouse_ua-0.3.0/src/dormouse_ua.egg-info/requires.txt +44 -0
  38. dormouse_ua-0.3.0/src/dormouse_ua.egg-info/top_level.txt +1 -0
  39. dormouse_ua-0.3.0/tests/__init__.py +0 -0
  40. dormouse_ua-0.3.0/tests/test_analyzer.py +40 -0
  41. dormouse_ua-0.3.0/tests/test_anthropic_proxy.py +527 -0
  42. dormouse_ua-0.3.0/tests/test_brew.py +215 -0
  43. dormouse_ua-0.3.0/tests/test_cli.py +44 -0
  44. dormouse_ua-0.3.0/tests/test_cli_search.py +42 -0
  45. dormouse_ua-0.3.0/tests/test_compressor.py +39 -0
  46. dormouse_ua-0.3.0/tests/test_db.py +74 -0
  47. dormouse_ua-0.3.0/tests/test_embedder.py +60 -0
  48. dormouse_ua-0.3.0/tests/test_import.py +177 -0
  49. dormouse_ua-0.3.0/tests/test_language_guard.py +129 -0
  50. dormouse_ua-0.3.0/tests/test_local_llm.py +116 -0
  51. dormouse_ua-0.3.0/tests/test_local_model.py +14 -0
  52. dormouse_ua-0.3.0/tests/test_mapper.py +36 -0
  53. dormouse_ua-0.3.0/tests/test_middleware.py +332 -0
  54. dormouse_ua-0.3.0/tests/test_middleware_e2e.py +141 -0
  55. dormouse_ua-0.3.0/tests/test_openrouter.py +21 -0
  56. dormouse_ua-0.3.0/tests/test_optimizer.py +55 -0
  57. dormouse_ua-0.3.0/tests/test_parsers.py +159 -0
  58. dormouse_ua-0.3.0/tests/test_rule_engine.py +105 -0
  59. dormouse_ua-0.3.0/tests/test_search.py +97 -0
  60. dormouse_ua-0.3.0/tests/test_seq2seq.py +76 -0
  61. dormouse_ua-0.3.0/tests/test_stream_buffer.py +88 -0
  62. dormouse_ua-0.3.0/tests/test_style_classifier.py +71 -0
  63. dormouse_ua-0.3.0/tests/test_teapot.py +248 -0
  64. dormouse_ua-0.3.0/tests/test_unsqueeze.py +139 -0
@@ -0,0 +1,17 @@
1
+ __pycache__/
2
+ *.py[cod]
3
+ *.egg-info/
4
+ dist/
5
+ build/
6
+ .venv/
7
+ .env
8
+ *.db
9
+ *.sqlite3
10
+ .pytest_cache/
11
+ .ruff_cache/
12
+ *.egg
13
+ .DS_Store
14
+ notebooks/.ipynb_checkpoints/
15
+ data/exports/*.jsonl
16
+ data/exports/*.csv
17
+ gemini.json
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Daria Chuprina
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,238 @@
1
+ Metadata-Version: 2.4
2
+ Name: dormouse-ua
3
+ Version: 0.3.0
4
+ Summary: Оптимізація українських текстів для LLM: менше токенів, краще розуміння
5
+ Author-email: Daria Chuprina <daria@lazysoft.dev>
6
+ License-Expression: MIT
7
+ Project-URL: Homepage, https://github.com/ChuprinaDaria/dormouse
8
+ Keywords: ukrainian,nlp,llm,tokenization,optimization
9
+ Classifier: Development Status :: 3 - Alpha
10
+ Classifier: Intended Audience :: Developers
11
+ Classifier: Programming Language :: Python :: 3.11
12
+ Classifier: Programming Language :: Python :: 3.12
13
+ Classifier: Topic :: Text Processing :: Linguistic
14
+ Requires-Python: >=3.11
15
+ Description-Content-Type: text/markdown
16
+ License-File: LICENSE
17
+ Requires-Dist: click>=8.1
18
+ Provides-Extra: morph
19
+ Requires-Dist: pymorphy3>=2.0; extra == "morph"
20
+ Provides-Extra: tokens
21
+ Requires-Dist: tiktoken>=0.7; extra == "tokens"
22
+ Requires-Dist: anthropic>=0.40; extra == "tokens"
23
+ Provides-Extra: embeddings
24
+ Requires-Dist: sentence-transformers>=3.0; extra == "embeddings"
25
+ Provides-Extra: compress
26
+ Requires-Dist: llmlingua>=0.2; extra == "compress"
27
+ Provides-Extra: translate
28
+ Requires-Dist: transformers>=4.40; extra == "translate"
29
+ Requires-Dist: sentencepiece>=0.2; extra == "translate"
30
+ Provides-Extra: ml
31
+ Requires-Dist: torch>=2.0; extra == "ml"
32
+ Provides-Extra: style
33
+ Requires-Dist: scikit-learn>=1.3; extra == "style"
34
+ Requires-Dist: numpy>=1.24; extra == "style"
35
+ Provides-Extra: search
36
+ Requires-Dist: sentence-transformers>=3.0; extra == "search"
37
+ Requires-Dist: openpyxl>=3.1; extra == "search"
38
+ Requires-Dist: pymupdf>=1.24; extra == "search"
39
+ Provides-Extra: middleware
40
+ Requires-Dist: openai>=1.0; extra == "middleware"
41
+ Provides-Extra: anthropic-middleware
42
+ Requires-Dist: anthropic>=0.40; extra == "anthropic-middleware"
43
+ Provides-Extra: all
44
+ Requires-Dist: dormouse[anthropic-middleware,compress,embeddings,middleware,ml,morph,search,style,tokens,translate]; extra == "all"
45
+ Provides-Extra: dev
46
+ Requires-Dist: pytest>=8.0; extra == "dev"
47
+ Requires-Dist: ruff>=0.5; extra == "dev"
48
+ Requires-Dist: pymorphy3>=2.0; extra == "dev"
49
+ Dynamic: license-file
50
+
51
+ # dormouse
52
+
53
+ **Ukrainian text optimizer for LLMs** — fewer tokens, better comprehension.
54
+
55
+ Normalizes surzhyk, slang, fillers, and maps to English for cloud LLMs. Saves 60-73% tokens while *improving* response quality.
56
+
57
+ > **UA:** Оптимізація українських текстів для LLM. Нормалізує суржик, сленг, мат — і стискає в англійську для Claude/GPT. Економія 60-73% токенів, якість відповідей зростає зі 67% до 100%.
58
+
59
+ ## Results
60
+
61
+ Tested on 53,351 texts (Telegram corpus + books), 12 IT prompts across 4 GPT models:
62
+
63
+ | Metric | Value |
64
+ |--------|-------|
65
+ | Token savings (cloud) | **73%** |
66
+ | Token savings (without seq2seq) | **49%** |
67
+ | Lexicon coverage | **88%** |
68
+ | Seq2seq exact match | **98.2%** |
69
+ | GPT response quality (original UA) | 67% |
70
+ | GPT response quality (squeezed EN) | **100%** |
71
+ | Quality preservation | **150%** (squeezed > original) |
72
+
73
+ ```
74
+ Original UA: "блін продакшн впав після деплою, що робити першим"
75
+ Squeezed EN: "damn production crashed after deploy, what do first"
76
+ Tokens: 45 → 12 (-73%)
77
+ GPT accuracy: 67% → 100%
78
+ ```
79
+
80
+ ## How it works
81
+
82
+ ```mermaid
83
+ graph LR
84
+ A[UA text<br/>surzhyk, slang] --> B[crack_open<br/>normalize]
85
+ B --> C[compress<br/>remove fillers]
86
+ C --> D[map_to_en<br/>lexicon + seq2seq]
87
+ D --> E[EN compressed<br/>for LLM]
88
+
89
+ style A fill:#fdd,stroke:#c33
90
+ style E fill:#dfd,stroke:#3a3
91
+ ```
92
+
93
+ | Layer | What it does | How |
94
+ |-------|-------------|-----|
95
+ | **crack_open** | surzhyk, slang, profanity → standard UA | 360 rules + pymorphy3 lemmatization |
96
+ | **compress** | remove fillers, intensifiers, noise | rule-based pattern matching |
97
+ | **map_to_en** | UA → compact English | 47K lexicon + seq2seq (28K expression pairs) |
98
+
99
+ ## Install
100
+
101
+ ```bash
102
+ pip install dormouse
103
+
104
+ # With morphological analysis (recommended)
105
+ pip install dormouse[morph]
106
+
107
+ # Everything
108
+ pip install dormouse[all]
109
+ ```
110
+
111
+ ## Quick start
112
+
113
+ ```python
114
+ from dormouse import squeeze
115
+
116
+ # Normalize only (layers 1+2)
117
+ squeeze("шо там по баґу, пофікси плз")
118
+ # → "що там по помилці, виправ"
119
+
120
+ # Cloud mode — compress for Claude/GPT (layers 1+2+3)
121
+ squeeze("ваще нормально, канєшно зробимо", target="cloud")
122
+ # → "generally ok, sure do"
123
+ ```
124
+
125
+ ### SDK Middleware (drop-in)
126
+
127
+ ```python
128
+ from openai import OpenAI
129
+ from dormouse import DormouseClient
130
+
131
+ client = DormouseClient(OpenAI()) # or Anthropic()
132
+
133
+ response = client.chat.completions.create(
134
+ model="gpt-4o-mini",
135
+ messages=[{"role": "user", "content": "шо там по деплою, він ваще не робе"}],
136
+ )
137
+ # Prompt: squeeze → EN → GPT → unsqueeze → Ukrainian response
138
+ ```
139
+
140
+ ### Semantic search
141
+
142
+ ```python
143
+ from dormouse import stir, mumble, sip
144
+
145
+ stir("report.pdf") # index
146
+ results = mumble("холодні закуски") # search by meaning
147
+ topics = sip("data.xlsx", topics=["HR", "finance"]) # classify
148
+ ```
149
+
150
+ ### CLI
151
+
152
+ ```bash
153
+ dormouse squeeze "шо там по баґу" -t cloud
154
+ dormouse stir book.pdf
155
+ dormouse mumble "головний герой"
156
+ ```
157
+
158
+ ## Comparison with alternatives
159
+
160
+ | Tool | Ukrainian | Token savings | Approach | Quality impact |
161
+ |------|:---------:|:------------:|----------|:--------------:|
162
+ | **dormouse** | native | **73%** | normalize + compress + translate | **+50% quality** |
163
+ | [LLMLingua](https://github.com/microsoft/LLMLingua) | no | up to 20x | ML perplexity pruning (GPT-2/LLaMA) | -5-15% |
164
+ | [Selective Context](https://github.com/liyucheng09/Selective_Context) | no | 40-50% | self-information filtering | -10-20% |
165
+ | [token-reducer](https://pypi.org/project/token-reducer/) | no | 50-75% | 6-stage pipeline, AST for code | neutral |
166
+ | [shrink-prompt](https://pypi.org/project/shrink-prompt/) | no | 30-70% | domain-specific rules (<20ms) | neutral |
167
+ | Google Translate → EN | partial | 30-40% | full translation | variable |
168
+
169
+ **Why dormouse is different:**
170
+
171
+ The problem: Ukrainian Cyrillic costs [3-4x more tokens](https://www.frontiersin.org/journals/artificial-intelligence/articles/10.3389/frai.2025.1538165/full) than equivalent English text in GPT-4/Claude.
172
+
173
+ All existing tools (LLMLingua, Selective Context, token-reducer) compress *already English* text by removing information. dormouse solves the problem one level earlier — transforms expensive Ukrainian (3-4 tokens/word) into cheap English (1-1.5 tokens/word) while **preserving all meaning**.
174
+
175
+ No other tool specifically optimizes Ukrainian for LLMs.
176
+
177
+ ## Use cases
178
+
179
+ **Cost reduction** — Ukrainian Cyrillic encodes into 2-4x more tokens than equivalent English. dormouse saves 60-73% on input tokens.
180
+
181
+ **Chatbots & support** — Users write in surzhyk/slang, dormouse normalizes before LLM, GPT gives concrete answers instead of generic responses.
182
+
183
+ **RAG & document search** — User searches in slang, documents are in literary language. dormouse normalizes both sides → finds by meaning.
184
+
185
+ **AI agents** — Long chains of actions eat context window. 73% compression = 73% more "memory" for the agent.
186
+
187
+ **Batch processing** — 10K comments through GPT for sentiment analysis. Squeeze first → cheaper and faster.
188
+
189
+ ## Eval details
190
+
191
+ Full evaluation ran for 4 days on 53,351 texts:
192
+
193
+ ```
194
+ Corpus: 53,351 texts (Telegram + books)
195
+ Squeeze speed: 606 texts/sec (normalization)
196
+ Seq2seq model: 7.3M params, 28K expression pairs
197
+ Stir/mumble: 8,441 chunks indexed, search ~600ms
198
+ Sip classification: 99% texts classified (8 topics)
199
+ ```
200
+
201
+ ### HF Inference API (small models)
202
+
203
+ | Model | UA score | Squeezed EN score | Delta |
204
+ |-------|:--------:|:-----------------:|:-----:|
205
+ | Qwen2.5-72B | 4.9/5 | 4.5/5 | -0.4 |
206
+ | Qwen2.5-7B | 4.4/5 | 3.6/5 | -0.8 |
207
+ | Llama-3.2-1B | 2.7/5 | 2.8/5 | +0.1 |
208
+
209
+ > Squeeze works great for cloud models (GPT, Claude, 70B+). For small models (<7B), use `brew()` with native Ukrainian — they understand UA better than squeezed EN.
210
+
211
+ ## Architecture
212
+
213
+ ```
214
+ src/dormouse/
215
+ ├── optimizer.py — squeeze() main pipeline
216
+ ├── rule_engine.py — normalization (360 rules + pymorphy3)
217
+ ├── compressor.py — filler/noise removal
218
+ ├── mapper.py — UA→EN via lexicon + lemma + transliteration
219
+ ├── seq2seq.py — expression translator (GRU encoder-decoder)
220
+ ├── teapot.py — stir/mumble/sip/brew (search + LLM)
221
+ ├── embedder.py — sentence-transformers wrapper
222
+ ├── middleware.py — OpenAI/Anthropic SDK proxy
223
+ ├── cli.py — Click CLI
224
+ └── assets.py — lazy download of models/data
225
+ ```
226
+
227
+ ## Development
228
+
229
+ ```bash
230
+ git clone https://github.com/ChuprinaDaria/dormouse
231
+ cd dormouse
232
+ pip install -e ".[dev,morph]"
233
+ DORMOUSE_DATA_DIR=./data pytest tests/ -v
234
+ ```
235
+
236
+ ## License
237
+
238
+ MIT
@@ -0,0 +1,188 @@
1
+ # dormouse
2
+
3
+ **Ukrainian text optimizer for LLMs** — fewer tokens, better comprehension.
4
+
5
+ Normalizes surzhyk, slang, fillers, and maps to English for cloud LLMs. Saves 60-73% tokens while *improving* response quality.
6
+
7
+ > **UA:** Оптимізація українських текстів для LLM. Нормалізує суржик, сленг, мат — і стискає в англійську для Claude/GPT. Економія 60-73% токенів, якість відповідей зростає зі 67% до 100%.
8
+
9
+ ## Results
10
+
11
+ Tested on 53,351 texts (Telegram corpus + books), 12 IT prompts across 4 GPT models:
12
+
13
+ | Metric | Value |
14
+ |--------|-------|
15
+ | Token savings (cloud) | **73%** |
16
+ | Token savings (without seq2seq) | **49%** |
17
+ | Lexicon coverage | **88%** |
18
+ | Seq2seq exact match | **98.2%** |
19
+ | GPT response quality (original UA) | 67% |
20
+ | GPT response quality (squeezed EN) | **100%** |
21
+ | Quality preservation | **150%** (squeezed > original) |
22
+
23
+ ```
24
+ Original UA: "блін продакшн впав після деплою, що робити першим"
25
+ Squeezed EN: "damn production crashed after deploy, what do first"
26
+ Tokens: 45 → 12 (-73%)
27
+ GPT accuracy: 67% → 100%
28
+ ```
29
+
30
+ ## How it works
31
+
32
+ ```mermaid
33
+ graph LR
34
+ A[UA text<br/>surzhyk, slang] --> B[crack_open<br/>normalize]
35
+ B --> C[compress<br/>remove fillers]
36
+ C --> D[map_to_en<br/>lexicon + seq2seq]
37
+ D --> E[EN compressed<br/>for LLM]
38
+
39
+ style A fill:#fdd,stroke:#c33
40
+ style E fill:#dfd,stroke:#3a3
41
+ ```
42
+
43
+ | Layer | What it does | How |
44
+ |-------|-------------|-----|
45
+ | **crack_open** | surzhyk, slang, profanity → standard UA | 360 rules + pymorphy3 lemmatization |
46
+ | **compress** | remove fillers, intensifiers, noise | rule-based pattern matching |
47
+ | **map_to_en** | UA → compact English | 47K lexicon + seq2seq (28K expression pairs) |
48
+
49
+ ## Install
50
+
51
+ ```bash
52
+ pip install dormouse
53
+
54
+ # With morphological analysis (recommended)
55
+ pip install dormouse[morph]
56
+
57
+ # Everything
58
+ pip install dormouse[all]
59
+ ```
60
+
61
+ ## Quick start
62
+
63
+ ```python
64
+ from dormouse import squeeze
65
+
66
+ # Normalize only (layers 1+2)
67
+ squeeze("шо там по баґу, пофікси плз")
68
+ # → "що там по помилці, виправ"
69
+
70
+ # Cloud mode — compress for Claude/GPT (layers 1+2+3)
71
+ squeeze("ваще нормально, канєшно зробимо", target="cloud")
72
+ # → "generally ok, sure do"
73
+ ```
74
+
75
+ ### SDK Middleware (drop-in)
76
+
77
+ ```python
78
+ from openai import OpenAI
79
+ from dormouse import DormouseClient
80
+
81
+ client = DormouseClient(OpenAI()) # or Anthropic()
82
+
83
+ response = client.chat.completions.create(
84
+ model="gpt-4o-mini",
85
+ messages=[{"role": "user", "content": "шо там по деплою, він ваще не робе"}],
86
+ )
87
+ # Prompt: squeeze → EN → GPT → unsqueeze → Ukrainian response
88
+ ```
89
+
90
+ ### Semantic search
91
+
92
+ ```python
93
+ from dormouse import stir, mumble, sip
94
+
95
+ stir("report.pdf") # index
96
+ results = mumble("холодні закуски") # search by meaning
97
+ topics = sip("data.xlsx", topics=["HR", "finance"]) # classify
98
+ ```
99
+
100
+ ### CLI
101
+
102
+ ```bash
103
+ dormouse squeeze "шо там по баґу" -t cloud
104
+ dormouse stir book.pdf
105
+ dormouse mumble "головний герой"
106
+ ```
107
+
108
+ ## Comparison with alternatives
109
+
110
+ | Tool | Ukrainian | Token savings | Approach | Quality impact |
111
+ |------|:---------:|:------------:|----------|:--------------:|
112
+ | **dormouse** | native | **73%** | normalize + compress + translate | **+50% quality** |
113
+ | [LLMLingua](https://github.com/microsoft/LLMLingua) | no | up to 20x | ML perplexity pruning (GPT-2/LLaMA) | -5-15% |
114
+ | [Selective Context](https://github.com/liyucheng09/Selective_Context) | no | 40-50% | self-information filtering | -10-20% |
115
+ | [token-reducer](https://pypi.org/project/token-reducer/) | no | 50-75% | 6-stage pipeline, AST for code | neutral |
116
+ | [shrink-prompt](https://pypi.org/project/shrink-prompt/) | no | 30-70% | domain-specific rules (<20ms) | neutral |
117
+ | Google Translate → EN | partial | 30-40% | full translation | variable |
118
+
119
+ **Why dormouse is different:**
120
+
121
+ The problem: Ukrainian Cyrillic costs [3-4x more tokens](https://www.frontiersin.org/journals/artificial-intelligence/articles/10.3389/frai.2025.1538165/full) than equivalent English text in GPT-4/Claude.
122
+
123
+ All existing tools (LLMLingua, Selective Context, token-reducer) compress *already English* text by removing information. dormouse solves the problem one level earlier — transforms expensive Ukrainian (3-4 tokens/word) into cheap English (1-1.5 tokens/word) while **preserving all meaning**.
124
+
125
+ No other tool specifically optimizes Ukrainian for LLMs.
126
+
127
+ ## Use cases
128
+
129
+ **Cost reduction** — Ukrainian Cyrillic encodes into 2-4x more tokens than equivalent English. dormouse saves 60-73% on input tokens.
130
+
131
+ **Chatbots & support** — Users write in surzhyk/slang, dormouse normalizes before LLM, GPT gives concrete answers instead of generic responses.
132
+
133
+ **RAG & document search** — User searches in slang, documents are in literary language. dormouse normalizes both sides → finds by meaning.
134
+
135
+ **AI agents** — Long chains of actions eat context window. 73% compression = 73% more "memory" for the agent.
136
+
137
+ **Batch processing** — 10K comments through GPT for sentiment analysis. Squeeze first → cheaper and faster.
138
+
139
+ ## Eval details
140
+
141
+ Full evaluation ran for 4 days on 53,351 texts:
142
+
143
+ ```
144
+ Corpus: 53,351 texts (Telegram + books)
145
+ Squeeze speed: 606 texts/sec (normalization)
146
+ Seq2seq model: 7.3M params, 28K expression pairs
147
+ Stir/mumble: 8,441 chunks indexed, search ~600ms
148
+ Sip classification: 99% texts classified (8 topics)
149
+ ```
150
+
151
+ ### HF Inference API (small models)
152
+
153
+ | Model | UA score | Squeezed EN score | Delta |
154
+ |-------|:--------:|:-----------------:|:-----:|
155
+ | Qwen2.5-72B | 4.9/5 | 4.5/5 | -0.4 |
156
+ | Qwen2.5-7B | 4.4/5 | 3.6/5 | -0.8 |
157
+ | Llama-3.2-1B | 2.7/5 | 2.8/5 | +0.1 |
158
+
159
+ > Squeeze works great for cloud models (GPT, Claude, 70B+). For small models (<7B), use `brew()` with native Ukrainian — they understand UA better than squeezed EN.
160
+
161
+ ## Architecture
162
+
163
+ ```
164
+ src/dormouse/
165
+ ├── optimizer.py — squeeze() main pipeline
166
+ ├── rule_engine.py — normalization (360 rules + pymorphy3)
167
+ ├── compressor.py — filler/noise removal
168
+ ├── mapper.py — UA→EN via lexicon + lemma + transliteration
169
+ ├── seq2seq.py — expression translator (GRU encoder-decoder)
170
+ ├── teapot.py — stir/mumble/sip/brew (search + LLM)
171
+ ├── embedder.py — sentence-transformers wrapper
172
+ ├── middleware.py — OpenAI/Anthropic SDK proxy
173
+ ├── cli.py — Click CLI
174
+ └── assets.py — lazy download of models/data
175
+ ```
176
+
177
+ ## Development
178
+
179
+ ```bash
180
+ git clone https://github.com/ChuprinaDaria/dormouse
181
+ cd dormouse
182
+ pip install -e ".[dev,morph]"
183
+ DORMOUSE_DATA_DIR=./data pytest tests/ -v
184
+ ```
185
+
186
+ ## License
187
+
188
+ MIT
@@ -0,0 +1,98 @@
1
+ [build-system]
2
+ requires = ["setuptools>=68.0", "setuptools-scm>=8.0"]
3
+ build-backend = "setuptools.build_meta"
4
+
5
+ [project]
6
+ name = "dormouse-ua"
7
+ version = "0.3.0"
8
+ description = "Оптимізація українських текстів для LLM: менше токенів, краще розуміння"
9
+ readme = "README.md"
10
+ license = "MIT"
11
+ requires-python = ">=3.11"
12
+ authors = [
13
+ {name = "Daria Chuprina", email = "daria@lazysoft.dev"},
14
+ ]
15
+ keywords = ["ukrainian", "nlp", "llm", "tokenization", "optimization"]
16
+ classifiers = [
17
+ "Development Status :: 3 - Alpha",
18
+ "Intended Audience :: Developers",
19
+ "Programming Language :: Python :: 3.11",
20
+ "Programming Language :: Python :: 3.12",
21
+ "Topic :: Text Processing :: Linguistic",
22
+ ]
23
+ dependencies = [
24
+ "click>=8.1",
25
+ ]
26
+
27
+ [project.optional-dependencies]
28
+ morph = [
29
+ "pymorphy3>=2.0",
30
+ ]
31
+ tokens = [
32
+ "tiktoken>=0.7",
33
+ "anthropic>=0.40",
34
+ ]
35
+ embeddings = [
36
+ "sentence-transformers>=3.0",
37
+ ]
38
+ compress = [
39
+ "llmlingua>=0.2",
40
+ ]
41
+ translate = [
42
+ "transformers>=4.40",
43
+ "sentencepiece>=0.2",
44
+ ]
45
+ ml = [
46
+ "torch>=2.0",
47
+ ]
48
+ style = [
49
+ "scikit-learn>=1.3",
50
+ "numpy>=1.24",
51
+ ]
52
+ search = [
53
+ "sentence-transformers>=3.0",
54
+ "openpyxl>=3.1",
55
+ "pymupdf>=1.24",
56
+ ]
57
+ middleware = [
58
+ "openai>=1.0",
59
+ ]
60
+ anthropic-middleware = [
61
+ "anthropic>=0.40",
62
+ ]
63
+ all = [
64
+ "dormouse[morph,tokens,embeddings,compress,translate,ml,style,search,middleware,anthropic-middleware]",
65
+ ]
66
+ dev = [
67
+ "pytest>=8.0",
68
+ "ruff>=0.5",
69
+ "pymorphy3>=2.0",
70
+ ]
71
+
72
+ [project.scripts]
73
+ dormouse = "dormouse.cli:main"
74
+
75
+ [project.urls]
76
+ Homepage = "https://github.com/ChuprinaDaria/dormouse"
77
+
78
+ [tool.setuptools.packages.find]
79
+ where = ["src"]
80
+
81
+ [tool.setuptools.package-data]
82
+ dormouse = ["data/*.json"]
83
+
84
+ [tool.ruff]
85
+ target-version = "py311"
86
+ line-length = 99
87
+
88
+ [tool.ruff.lint]
89
+ select = ["E", "F", "I", "W"]
90
+
91
+ [tool.ruff.lint.per-file-ignores]
92
+ "tests/*" = ["E402"]
93
+
94
+ [tool.pytest.ini_options]
95
+ testpaths = ["tests"]
96
+ markers = [
97
+ "e2e: end-to-end tests with real API (require API keys)",
98
+ ]
@@ -0,0 +1,4 @@
1
+ [egg_info]
2
+ tag_build =
3
+ tag_date = 0
4
+
@@ -0,0 +1,95 @@
1
+ """dormouse — оптимізація українських текстів для LLM."""
2
+
3
+ __version__ = "0.3.0"
4
+
5
+ from dormouse.optimizer import SqueezedText, squeeze, squeeze_batch
6
+
7
+ __all__ = [
8
+ "squeeze",
9
+ "squeeze_batch",
10
+ "SqueezedText",
11
+ "__version__",
12
+ ]
13
+
14
+
15
+ def nibble(text: str):
16
+ """Бенчмарк токенізації (потребує pip install dormouse[tokens])."""
17
+ from dormouse.analyzer import nibble as _nibble
18
+
19
+ return _nibble(text)
20
+
21
+
22
+ def nibble_batch(texts: list[str]):
23
+ """Батч-аналіз токенізації."""
24
+ from dormouse.analyzer import nibble_batch as _nibble_batch
25
+
26
+ return _nibble_batch(texts)
27
+
28
+
29
+ def sniff(
30
+ items: list[str],
31
+ categories: list[str],
32
+ squeeze_first: bool = False,
33
+ ):
34
+ """Класифікація через embeddings (потребує pip install dormouse[embeddings])."""
35
+ from dormouse.classifier import sniff as _sniff
36
+
37
+ return _sniff(items, categories, squeeze_first=squeeze_first)
38
+
39
+
40
+ def DormouseClient(client, **kwargs):
41
+ """SDK wrapper для OpenAI (потребує pip install dormouse[middleware])."""
42
+ from dormouse.middleware import DormouseClient as _DormouseClient
43
+
44
+ return _DormouseClient(client, **kwargs)
45
+
46
+
47
+ def stir(source, **kwargs):
48
+ """Завантажити і проіндексувати файл (потребує pip install dormouse[search])."""
49
+ from dormouse.teapot import Teapot
50
+
51
+ try:
52
+ from dormouse.embedder import Embedder
53
+
54
+ emb = Embedder()
55
+ except ImportError:
56
+ emb = None
57
+ t = Teapot(embedder=emb)
58
+ return t.stir(source, **kwargs)
59
+
60
+
61
+ def mumble(query, index=None, top_k=10):
62
+ """Семантичний пошук по проіндексованих даних."""
63
+ from dormouse.teapot import Teapot
64
+
65
+ if index is not None:
66
+ return index.mumble(query, top_k=top_k)
67
+ t = Teapot()
68
+ return t.mumble(query, top_k=top_k)
69
+
70
+
71
+ def sip(source, topics, threshold=0.5, **kwargs):
72
+ """Класифікація тексту по заданих темах."""
73
+ from dormouse.embedder import Embedder
74
+ from dormouse.teapot import Teapot
75
+
76
+ emb = Embedder()
77
+ t = Teapot(embedder=emb)
78
+ return t.sip(source, topics, threshold=threshold)
79
+
80
+
81
+ def brew(query, *, llm, index=None, extract=None, strategy="hierarchical", limit=50):
82
+ """LLM-powered пошук і витягування (потребує pip install dormouse[search])."""
83
+ from dormouse.teapot import Teapot
84
+
85
+ if index is not None:
86
+ return index.brew(query, llm=llm, extract=extract, strategy=strategy, limit=limit)
87
+ t = Teapot()
88
+ return t.brew(query, llm=llm, extract=extract, strategy=strategy, limit=limit)
89
+
90
+
91
+ def LocalLLM(model="MamayLM-4B", **kwargs):
92
+ """Уніфікований інтерфейс до локальної LLM."""
93
+ from dormouse.local_model import LocalLLM as _LocalLLM
94
+
95
+ return _LocalLLM(model, **kwargs)