datoon 1.7.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- datoon/__init__.py +31 -0
- datoon/_version.py +24 -0
- datoon/analyzer.py +89 -0
- datoon/cli.py +197 -0
- datoon/converter.py +165 -0
- datoon/mcp_server.py +140 -0
- datoon/models.py +65 -0
- datoon/py.typed +0 -0
- datoon/readers/__init__.py +98 -0
- datoon/readers/columnar.py +42 -0
- datoon/readers/csv.py +35 -0
- datoon/readers/excel.py +31 -0
- datoon/readers/jsonl.py +27 -0
- datoon/readers/numbers.py +30 -0
- datoon/readers/xml.py +58 -0
- datoon/readers/yaml.py +30 -0
- datoon-1.7.0.dist-info/METADATA +544 -0
- datoon-1.7.0.dist-info/RECORD +24 -0
- datoon-1.7.0.dist-info/WHEEL +5 -0
- datoon-1.7.0.dist-info/entry_points.txt +3 -0
- datoon-1.7.0.dist-info/licenses/LICENSE +21 -0
- datoon-1.7.0.dist-info/scm_file_list.json +117 -0
- datoon-1.7.0.dist-info/scm_version.json +8 -0
- datoon-1.7.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,544 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: datoon
|
|
3
|
+
Version: 1.7.0
|
|
4
|
+
Summary: Smart structured-data-to-TOON gateway with pragmatic auto-gating for LLM prompts.
|
|
5
|
+
Author-email: Andrii Suruhov <andrii.suruhov@gmail.com>
|
|
6
|
+
License: MIT License
|
|
7
|
+
|
|
8
|
+
Copyright (c) 2026 Andrii Suruhov
|
|
9
|
+
|
|
10
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
11
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
12
|
+
in the Software without restriction, including without limitation the rights
|
|
13
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
14
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
15
|
+
furnished to do so, subject to the following conditions:
|
|
16
|
+
|
|
17
|
+
The above copyright notice and this permission notice shall be included in all
|
|
18
|
+
copies or substantial portions of the Software.
|
|
19
|
+
|
|
20
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
21
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
22
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
23
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
24
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
25
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
26
|
+
SOFTWARE.
|
|
27
|
+
|
|
28
|
+
Project-URL: Homepage, https://github.com/andrii-su/datoon
|
|
29
|
+
Project-URL: Repository, https://github.com/andrii-su/datoon
|
|
30
|
+
Project-URL: Issues, https://github.com/andrii-su/datoon/issues
|
|
31
|
+
Keywords: llm,prompt-engineering,json,toon,csv,yaml,parquet,data
|
|
32
|
+
Classifier: Programming Language :: Python :: 3
|
|
33
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
34
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
35
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
36
|
+
Classifier: Operating System :: OS Independent
|
|
37
|
+
Requires-Python: >=3.12
|
|
38
|
+
Description-Content-Type: text/markdown
|
|
39
|
+
License-File: LICENSE
|
|
40
|
+
Provides-Extra: dev
|
|
41
|
+
Requires-Dist: pytest>=9.0.3; extra == "dev"
|
|
42
|
+
Requires-Dist: pre-commit>=4.6.0; extra == "dev"
|
|
43
|
+
Requires-Dist: pyyaml>=6.0; extra == "dev"
|
|
44
|
+
Provides-Extra: tokens
|
|
45
|
+
Requires-Dist: tiktoken>=0.13.0; extra == "tokens"
|
|
46
|
+
Provides-Extra: mcp
|
|
47
|
+
Requires-Dist: mcp>=1.28.0; extra == "mcp"
|
|
48
|
+
Provides-Extra: yaml
|
|
49
|
+
Requires-Dist: pyyaml>=6.0; extra == "yaml"
|
|
50
|
+
Provides-Extra: excel
|
|
51
|
+
Requires-Dist: openpyxl>=3.1; extra == "excel"
|
|
52
|
+
Provides-Extra: columnar
|
|
53
|
+
Requires-Dist: pyarrow>=15.0; extra == "columnar"
|
|
54
|
+
Requires-Dist: fastavro>=1.9; extra == "columnar"
|
|
55
|
+
Provides-Extra: numbers
|
|
56
|
+
Requires-Dist: numbers-parser>=4.4; extra == "numbers"
|
|
57
|
+
Provides-Extra: all
|
|
58
|
+
Requires-Dist: datoon[columnar,excel,mcp,numbers,tokens,yaml]; extra == "all"
|
|
59
|
+
Dynamic: license-file
|
|
60
|
+
|
|
61
|
+
<p align="center">
|
|
62
|
+
<img src="assets/datoon-banner.svg" width="720" />
|
|
63
|
+
</p>
|
|
64
|
+
|
|
65
|
+
<h1 align="center">datoon</h1>
|
|
66
|
+
|
|
67
|
+
<p align="center">
|
|
68
|
+
<strong>smart structured-data→TOON gateway — converts only when it actually saves tokens</strong>
|
|
69
|
+
</p>
|
|
70
|
+
|
|
71
|
+
<p align="center">
|
|
72
|
+
<a href="https://github.com/andrii-su/datoon/actions/workflows/tests.yml"><img src="https://github.com/andrii-su/datoon/actions/workflows/tests.yml/badge.svg" alt="Tests"></a>
|
|
73
|
+
<a href="https://github.com/andrii-su/datoon/actions/workflows/pre-commit.yml"><img src="https://github.com/andrii-su/datoon/actions/workflows/pre-commit.yml/badge.svg" alt="Pre-commit"></a>
|
|
74
|
+
<a href="https://github.com/andrii-su/datoon/actions/workflows/release.yml"><img src="https://github.com/andrii-su/datoon/actions/workflows/release.yml/badge.svg" alt="Release"></a>
|
|
75
|
+
<a href="https://pypi.org/project/datoon/"><img src="https://img.shields.io/pypi/v/datoon?color=0EA5E9" alt="PyPI"></a>
|
|
76
|
+
<a href="https://img.shields.io/badge/python-3.12%2B-3776AB"><img src="https://img.shields.io/badge/python-3.12%2B-3776AB?logo=python&logoColor=white" alt="Python"></a>
|
|
77
|
+
<a href="./LICENSE"><img src="https://img.shields.io/github/license/andrii-su/datoon" alt="License: MIT"></a>
|
|
78
|
+
</p>
|
|
79
|
+
|
|
80
|
+
<p align="center">
|
|
81
|
+
<a href="#before--after">Before/After</a> •
|
|
82
|
+
<a href="#install">Install</a> •
|
|
83
|
+
<a href="#what-you-get">What You Get</a> •
|
|
84
|
+
<a href="#how-it-works">How It Works</a> •
|
|
85
|
+
<a href="#benchmarks">Benchmarks</a> •
|
|
86
|
+
<a href="./INSTALL.md">Full install guide</a>
|
|
87
|
+
</p>
|
|
88
|
+
|
|
89
|
+
______________________________________________________________________
|
|
90
|
+
|
|
91
|
+
Raw structured data is often verbose in LLM prompts. [TOON](https://github.com/toon-format/toon) can save tokens — but blind conversion can also make payloads *worse*. `datoon` adds a decision layer: convert when structure and savings justify it, skip when they don't, and always explain why.
|
|
92
|
+
|
|
93
|
+
Supports **JSON, CSV, JSONL, YAML, XML, Parquet, Avro, ORC, Excel, and Apple Numbers** — auto-detected from file extension.
|
|
94
|
+
|
|
95
|
+
## Before / After
|
|
96
|
+
|
|
97
|
+
<table>
|
|
98
|
+
<tr>
|
|
99
|
+
<td width="50%">
|
|
100
|
+
|
|
101
|
+
### JSON in the prompt (43 tokens)
|
|
102
|
+
|
|
103
|
+
```json
|
|
104
|
+
{"users":[
|
|
105
|
+
{"id":1,"name":"Ada","role":"admin"},
|
|
106
|
+
{"id":2,"name":"Lin","role":"analyst"},
|
|
107
|
+
{"id":3,"name":"Grace","role":"viewer"}
|
|
108
|
+
]}
|
|
109
|
+
```
|
|
110
|
+
|
|
111
|
+
</td>
|
|
112
|
+
<td width="50%">
|
|
113
|
+
|
|
114
|
+
### datoon converts → TOON (24 tokens)
|
|
115
|
+
|
|
116
|
+
```toon
|
|
117
|
+
users[3]{id,name,role}:
|
|
118
|
+
1,Ada,admin
|
|
119
|
+
2,Lin,analyst
|
|
120
|
+
3,Grace,viewer
|
|
121
|
+
```
|
|
122
|
+
|
|
123
|
+
```json
|
|
124
|
+
{"decision":"convert","reason":"Estimated savings 44.19% (threshold 15.00%)."}
|
|
125
|
+
```
|
|
126
|
+
|
|
127
|
+
</td>
|
|
128
|
+
</tr>
|
|
129
|
+
<tr>
|
|
130
|
+
<td>
|
|
131
|
+
|
|
132
|
+
### CSV from a data pipeline (111 tokens as JSON)
|
|
133
|
+
|
|
134
|
+
```csv
|
|
135
|
+
id,name,role
|
|
136
|
+
1,Ada,admin
|
|
137
|
+
2,Lin,analyst
|
|
138
|
+
3,Grace,viewer
|
|
139
|
+
```
|
|
140
|
+
|
|
141
|
+
</td>
|
|
142
|
+
<td>
|
|
143
|
+
|
|
144
|
+
### datoon auto-converts → TOON (24 tokens)
|
|
145
|
+
|
|
146
|
+
```bash
|
|
147
|
+
datoon data.csv --report-stdout
|
|
148
|
+
```
|
|
149
|
+
|
|
150
|
+
Same result. Zero JSON serialization in your code.
|
|
151
|
+
|
|
152
|
+
</td>
|
|
153
|
+
</tr>
|
|
154
|
+
<tr>
|
|
155
|
+
<td>
|
|
156
|
+
|
|
157
|
+
### Non-uniform payload (26 tokens)
|
|
158
|
+
|
|
159
|
+
```json
|
|
160
|
+
{"config":{"debug":true},"tags":["a","b"]}
|
|
161
|
+
```
|
|
162
|
+
|
|
163
|
+
</td>
|
|
164
|
+
<td>
|
|
165
|
+
|
|
166
|
+
### datoon skips → keeps JSON
|
|
167
|
+
|
|
168
|
+
```json
|
|
169
|
+
{"decision":"skip","reason":"No uniform object arrays found with at least 3 rows."}
|
|
170
|
+
```
|
|
171
|
+
|
|
172
|
+
No Node.js call. No silent corruption.
|
|
173
|
+
|
|
174
|
+
</td>
|
|
175
|
+
</tr>
|
|
176
|
+
</table>
|
|
177
|
+
|
|
178
|
+
**Same data. Right format. Always explained.**
|
|
179
|
+
|
|
180
|
+
```
|
|
181
|
+
┌──────────────────────────────────────────────────┐
|
|
182
|
+
│ PAYLOAD SAVINGS (auto avg) ████░░░░░░ 28% │
|
|
183
|
+
│ PAYLOAD SAVINGS (agent skill) ████████░░ 62% │
|
|
184
|
+
│ DECISION ACCURACY ██████████ 100% │
|
|
185
|
+
│ HARMFUL CONVERSIONS BLOCKED ██████████ 100% │
|
|
186
|
+
└──────────────────────────────────────────────────┘
|
|
187
|
+
```
|
|
188
|
+
|
|
189
|
+
> [!IMPORTANT]
|
|
190
|
+
> datoon saves **payload tokens** — the structured data portion of your prompt. Token savings depend on payload shape: uniform tabular data converts well; deeply nested or non-uniform structures are skipped. Every decision includes a reason so pipelines can log, debug, and trust the outcome.
|
|
191
|
+
|
|
192
|
+
## Install
|
|
193
|
+
|
|
194
|
+
```bash
|
|
195
|
+
# core (JSON, CSV, JSONL, XML — no extra deps)
|
|
196
|
+
uv add datoon
|
|
197
|
+
pip install datoon
|
|
198
|
+
|
|
199
|
+
# with YAML support
|
|
200
|
+
pip install "datoon[yaml]"
|
|
201
|
+
|
|
202
|
+
# with Excel support
|
|
203
|
+
pip install "datoon[excel]"
|
|
204
|
+
|
|
205
|
+
# with Parquet / ORC / Avro support
|
|
206
|
+
pip install "datoon[columnar]"
|
|
207
|
+
|
|
208
|
+
# with Apple Numbers support
|
|
209
|
+
pip install "datoon[numbers]"
|
|
210
|
+
|
|
211
|
+
# with tiktoken-based token counting
|
|
212
|
+
pip install "datoon[tokens]"
|
|
213
|
+
|
|
214
|
+
# with MCP server
|
|
215
|
+
pip install "datoon[mcp]"
|
|
216
|
+
|
|
217
|
+
# everything
|
|
218
|
+
pip install "datoon[all]"
|
|
219
|
+
```
|
|
220
|
+
|
|
221
|
+
Requires Python 3.12+. TOON conversion requires Node.js with `npx` in PATH — analysis and format reading work without it.
|
|
222
|
+
|
|
223
|
+
For Claude Code plugin, Codex, and MCP config → [**INSTALL.md**](./INSTALL.md).
|
|
224
|
+
|
|
225
|
+
## What You Get
|
|
226
|
+
|
|
227
|
+
| | What |
|
|
228
|
+
|---|---|
|
|
229
|
+
| `datoon` **CLI** | Auto-gate any supported format → TOON from terminal or scripts |
|
|
230
|
+
| **Python API** | `convert_json_for_llm()` + `read_tabular()` for any LLM pipeline |
|
|
231
|
+
| **MCP Server** | `convert_json`, `convert_text`, `analyze_json` tools for Claude Desktop, Cursor, Windsurf |
|
|
232
|
+
| **Claude Code Plugin** | `/datoon` in-session trigger, installs from GitHub in one command |
|
|
233
|
+
| **Codex Plugin** | Marketplace plugin — structured-data mode for Codex |
|
|
234
|
+
|
|
235
|
+
### Supported input formats
|
|
236
|
+
|
|
237
|
+
| Format | Extension | Extra needed |
|
|
238
|
+
|---|---|---|
|
|
239
|
+
| JSON | `.json` | — |
|
|
240
|
+
| JSONL | `.jsonl`, `.ndjson` | — |
|
|
241
|
+
| CSV | `.csv` | — |
|
|
242
|
+
| XML | `.xml` | — |
|
|
243
|
+
| YAML | `.yaml`, `.yml` | `datoon[yaml]` |
|
|
244
|
+
| Excel | `.xlsx`, `.xls` | `datoon[excel]` |
|
|
245
|
+
| Parquet | `.parquet` | `datoon[columnar]` |
|
|
246
|
+
| Avro | `.avro` | `datoon[columnar]` |
|
|
247
|
+
| ORC | `.orc` | `datoon[columnar]` |
|
|
248
|
+
| Apple Numbers | `.numbers` | `datoon[numbers]` |
|
|
249
|
+
|
|
250
|
+
## How It Works
|
|
251
|
+
|
|
252
|
+
1. **Detect format** — from `--format` flag, file extension, or default to JSON for stdin
|
|
253
|
+
1. **Read + normalize** — parse source into list of row dicts; serialize to compact JSON
|
|
254
|
+
1. **Analyze structure** — uniform object arrays? acceptable depth? minimum rows?
|
|
255
|
+
1. **Gate early** — non-candidates skip before any CLI call; no Node.js overhead
|
|
256
|
+
1. **Convert + estimate** — TOON CLI runs, token savings calculated
|
|
257
|
+
1. **Gate savings** — below threshold → return JSON; above → return TOON with report
|
|
258
|
+
|
|
259
|
+
Every path returns a `ConversionReport` with `decision`, `reason`, and token estimates. Pipelines never get silent surprises.
|
|
260
|
+
|
|
261
|
+
______________________________________________________________________
|
|
262
|
+
|
|
263
|
+
## Quick Start
|
|
264
|
+
|
|
265
|
+
**JSON (stdin):**
|
|
266
|
+
|
|
267
|
+
```bash
|
|
268
|
+
echo '{"users":[{"id":1,"name":"Ada"},{"id":2,"name":"Lin"},{"id":3,"name":"Grace"}]}' | datoon --report-stdout
|
|
269
|
+
```
|
|
270
|
+
|
|
271
|
+
**CSV (auto-detected from extension):**
|
|
272
|
+
|
|
273
|
+
```bash
|
|
274
|
+
datoon data.csv --report-stdout
|
|
275
|
+
```
|
|
276
|
+
|
|
277
|
+
**JSONL:**
|
|
278
|
+
|
|
279
|
+
```bash
|
|
280
|
+
datoon data.jsonl -o output.toon
|
|
281
|
+
```
|
|
282
|
+
|
|
283
|
+
**YAML (requires `datoon[yaml]`):**
|
|
284
|
+
|
|
285
|
+
```bash
|
|
286
|
+
datoon data.yaml --report-stdout
|
|
287
|
+
```
|
|
288
|
+
|
|
289
|
+
**Parquet (requires `datoon[columnar]`):**
|
|
290
|
+
|
|
291
|
+
```bash
|
|
292
|
+
datoon data.parquet --report ./report.json
|
|
293
|
+
```
|
|
294
|
+
|
|
295
|
+
**Explicit format override:**
|
|
296
|
+
|
|
297
|
+
```bash
|
|
298
|
+
datoon --format csv < data.csv --report-stdout
|
|
299
|
+
```
|
|
300
|
+
|
|
301
|
+
**Force conversion (bypass gating — for experiments):**
|
|
302
|
+
|
|
303
|
+
```bash
|
|
304
|
+
datoon data.json --force --report-stdout
|
|
305
|
+
```
|
|
306
|
+
|
|
307
|
+
______________________________________________________________________
|
|
308
|
+
|
|
309
|
+
## Python API
|
|
310
|
+
|
|
311
|
+
**JSON conversion:**
|
|
312
|
+
|
|
313
|
+
```python
|
|
314
|
+
from datoon import convert_json_for_llm, ConversionConfig, DatoonError
|
|
315
|
+
|
|
316
|
+
config = ConversionConfig(min_savings_ratio=0.15, max_depth=6, min_uniform_rows=3)
|
|
317
|
+
|
|
318
|
+
try:
|
|
319
|
+
outcome = convert_json_for_llm(raw_json, config)
|
|
320
|
+
except DatoonError as exc:
|
|
321
|
+
raise
|
|
322
|
+
|
|
323
|
+
# outcome.payload_text — TOON or original JSON
|
|
324
|
+
# outcome.report.decision — "convert" | "skip"
|
|
325
|
+
# outcome.report.reason — human-readable explanation
|
|
326
|
+
send_to_model(outcome.payload_text)
|
|
327
|
+
```
|
|
328
|
+
|
|
329
|
+
**Any format via `read_tabular`:**
|
|
330
|
+
|
|
331
|
+
```python
|
|
332
|
+
import json
|
|
333
|
+
from pathlib import Path
|
|
334
|
+
from datoon import read_tabular, convert_json_for_llm, ConversionConfig
|
|
335
|
+
|
|
336
|
+
# text formats: csv, jsonl, yaml, xml
|
|
337
|
+
rows = read_tabular("csv", text=csv_string)
|
|
338
|
+
|
|
339
|
+
# binary formats: excel, parquet, orc, avro, numbers
|
|
340
|
+
rows = read_tabular("parquet", path=Path("data.parquet"))
|
|
341
|
+
|
|
342
|
+
json_text = json.dumps(rows, separators=(",", ":"))
|
|
343
|
+
outcome = convert_json_for_llm(json_text, ConversionConfig())
|
|
344
|
+
send_to_model(outcome.payload_text)
|
|
345
|
+
```
|
|
346
|
+
|
|
347
|
+
**Structure-only analysis (no Node.js required):**
|
|
348
|
+
|
|
349
|
+
```python
|
|
350
|
+
from datoon.analyzer import analyze_payload
|
|
351
|
+
from datoon.models import ConversionConfig
|
|
352
|
+
|
|
353
|
+
analysis = analyze_payload(parsed_data, ConversionConfig())
|
|
354
|
+
print(analysis.is_candidate, analysis.reason)
|
|
355
|
+
```
|
|
356
|
+
|
|
357
|
+
______________________________________________________________________
|
|
358
|
+
|
|
359
|
+
## MCP Server
|
|
360
|
+
|
|
361
|
+
<!-- mcp-name: io.github.andrii-su/datoon -->
|
|
362
|
+
|
|
363
|
+
`datoon` ships an [MCP](https://modelcontextprotocol.io) server with three tools:
|
|
364
|
+
|
|
365
|
+
| Tool | Description |
|
|
366
|
+
|---|---|
|
|
367
|
+
| `convert_json` | Full JSON conversion with policy gating |
|
|
368
|
+
| `convert_text` | Converts CSV, YAML, XML, or JSONL text with policy gating |
|
|
369
|
+
| `analyze_json` | Structure analysis only — no Node.js needed |
|
|
370
|
+
|
|
371
|
+
**Claude Desktop / Cursor / Windsurf config:**
|
|
372
|
+
|
|
373
|
+
```json
|
|
374
|
+
{
|
|
375
|
+
"mcpServers": {
|
|
376
|
+
"datoon": {
|
|
377
|
+
"command": "uvx",
|
|
378
|
+
"args": ["--from", "datoon[mcp]", "datoon", "mcp"]
|
|
379
|
+
}
|
|
380
|
+
}
|
|
381
|
+
}
|
|
382
|
+
```
|
|
383
|
+
|
|
384
|
+
**Run locally:**
|
|
385
|
+
|
|
386
|
+
```bash
|
|
387
|
+
datoon mcp # or the standalone script: datoon-mcp
|
|
388
|
+
```
|
|
389
|
+
|
|
390
|
+
Listed on the [MCP Registry](https://registry.modelcontextprotocol.io), [Smithery](https://smithery.ai), and [Glama](https://glama.ai). See [MARKETPLACES.md](./MARKETPLACES.md).
|
|
391
|
+
|
|
392
|
+
______________________________________________________________________
|
|
393
|
+
|
|
394
|
+
## Claude Code Plugin
|
|
395
|
+
|
|
396
|
+
Install directly from GitHub:
|
|
397
|
+
|
|
398
|
+
```bash
|
|
399
|
+
claude plugin marketplace add andrii-su/datoon
|
|
400
|
+
claude plugin install datoon@datoon
|
|
401
|
+
```
|
|
402
|
+
|
|
403
|
+
Trigger in-session:
|
|
404
|
+
|
|
405
|
+
```
|
|
406
|
+
/datoon
|
|
407
|
+
convert this JSON to TOON if it saves tokens
|
|
408
|
+
use datoon mode for structured data
|
|
409
|
+
```
|
|
410
|
+
|
|
411
|
+
______________________________________________________________________
|
|
412
|
+
|
|
413
|
+
## CLI Reference
|
|
414
|
+
|
|
415
|
+
| Flag | Default | Description |
|
|
416
|
+
|---|---|---|
|
|
417
|
+
| `--format` | auto | Input format: `json`, `csv`, `jsonl`, `yaml`, `xml`, `excel`, `parquet`, `avro`, `orc`, `numbers` |
|
|
418
|
+
| `--force` | `false` | Bypass gating and minimum savings threshold |
|
|
419
|
+
| `--min-savings` | `0.15` | Minimum relative token savings required |
|
|
420
|
+
| `--max-depth` | `6` | Maximum nesting depth for auto-conversion |
|
|
421
|
+
| `--min-uniform-rows` | `3` | Minimum rows in uniform object arrays |
|
|
422
|
+
| `--timeout` | `30` | Seconds before TOON CLI call is aborted |
|
|
423
|
+
| `--report <path>` | — | Write JSON conversion report to file |
|
|
424
|
+
| `--report-stdout` | — | Print JSON conversion report to stderr |
|
|
425
|
+
| `-o <path>` | stdout | Output file path |
|
|
426
|
+
| `--version` | — | Print version and exit |
|
|
427
|
+
|
|
428
|
+
Format is auto-detected from file extension. Use `--format` to override or when reading from stdin.
|
|
429
|
+
|
|
430
|
+
______________________________________________________________________
|
|
431
|
+
|
|
432
|
+
## Benchmarks
|
|
433
|
+
|
|
434
|
+
```bash
|
|
435
|
+
PYTHONPATH=src python benchmarks/run.py --dry-run
|
|
436
|
+
PYTHONPATH=src python benchmarks/run.py
|
|
437
|
+
PYTHONPATH=src python benchmarks/run.py --update-readme
|
|
438
|
+
```
|
|
439
|
+
|
|
440
|
+
### Why auto mode outperforms forced conversion
|
|
441
|
+
|
|
442
|
+
Auto mode avoids low-benefit and high-risk payloads (`orders-nested`, `mixed-non-uniform`) while matching forced TOON's average token count on suitable ones. Every decision comes with a reasoned report.
|
|
443
|
+
|
|
444
|
+
| Scenario | JSON Baseline | Forced TOON | `datoon` Auto |
|
|
445
|
+
|---|---:|---:|---:|
|
|
446
|
+
| Average tokens | 77 | 50 | 50 |
|
|
447
|
+
| Avg token saved | 0.0% | 26.8% | **28.1%** |
|
|
448
|
+
| Decision quality | n/a | Converts all | Converts `3/5`, skips harmful cases |
|
|
449
|
+
|
|
450
|
+
<!-- BENCHMARK-TABLE-START -->
|
|
451
|
+
|
|
452
|
+
| Dataset | JSON | TOON (forced) | Raw Saved | Auto | Auto Tokens | Auto Saved |
|
|
453
|
+
|---|---:|---:|---:|---|---:|---:|
|
|
454
|
+
| users-small | 56 | 31 | 44.6% | convert | 31 | 44.6% |
|
|
455
|
+
| events-medium | 198 | 111 | 43.9% | convert | 111 | 43.9% |
|
|
456
|
+
| orders-nested | 93 | 91 | 2.2% | skip | 93 | 0.0% |
|
|
457
|
+
| mixed-non-uniform | 35 | 37 | -5.7% | skip | 35 | 0.0% |
|
|
458
|
+
| metrics-wide | 133 | 63 | 52.6% | convert | 63 | 52.6% |
|
|
459
|
+
| **Average** | **103** | **67** | **27.5%** | **3/5 convert** | **67** | **28.2%** |
|
|
460
|
+
|
|
461
|
+
*Forced conversion succeeded for 5/5 payloads.*
|
|
462
|
+
|
|
463
|
+
<!-- BENCHMARK-TABLE-END -->
|
|
464
|
+
|
|
465
|
+
### Format conversion benchmark
|
|
466
|
+
|
|
467
|
+
Token savings when converting from common structured formats (CSV, JSONL, XML, YAML).
|
|
468
|
+
Baseline is the JSON representation of the same data — what an LLM would receive without datoon.
|
|
469
|
+
|
|
470
|
+
<!-- FORMAT-BENCHMARK-TABLE-START -->
|
|
471
|
+
|
|
472
|
+
| Dataset | Format | JSON Tokens | TOON (forced) | Auto | Auto Tokens | Auto Saved |
|
|
473
|
+
|---|---|---:|---:|---|---:|---:|
|
|
474
|
+
| users-csv | csv | 53 | 29 | convert | 29 | 45.3% |
|
|
475
|
+
| events-jsonl | jsonl | 194 | 109 | convert | 109 | 43.8% |
|
|
476
|
+
| catalog-xml | xml | 96 | 50 | convert | 50 | 47.9% |
|
|
477
|
+
| metrics-yaml | yaml | 129 | 61 | convert | 61 | 52.7% |
|
|
478
|
+
| **Average** | — | **118** | **62** | **4/4 convert** | **62** | **47.4%** |
|
|
479
|
+
|
|
480
|
+
*Forced conversion succeeded for 4/4 payloads.*
|
|
481
|
+
|
|
482
|
+
<!-- FORMAT-BENCHMARK-TABLE-END -->
|
|
483
|
+
|
|
484
|
+
### Agent skill evaluation
|
|
485
|
+
|
|
486
|
+
Artifact-based subagent comparison — identical analysis tasks, two modes:
|
|
487
|
+
|
|
488
|
+
- `with_skill`: agent received the `datoon` skill and followed the conversion workflow.
|
|
489
|
+
- `without_skill`: agent used JSON directly, no TOON or `datoon`.
|
|
490
|
+
|
|
491
|
+
3 payload sizes × 3 iterations = 18 total agent runs. Both modes: 100% correct answers.
|
|
492
|
+
|
|
493
|
+
| Scenario | Avg JSON Tokens | Avg TOON Tokens | Avg Payload Saved |
|
|
494
|
+
|---|---:|---:|---:|
|
|
495
|
+
| small | 225 | 118 | 47.6% |
|
|
496
|
+
| medium | 2,972 | 1,138 | 61.7% |
|
|
497
|
+
| large | 17,757 | 6,673 | 62.4% |
|
|
498
|
+
|
|
499
|
+
Full report and raw outputs: [`benchmarks/agent_skill_eval/`](benchmarks/agent_skill_eval/). Savings are payload-token estimates, not full end-to-end model-token usage.
|
|
500
|
+
|
|
501
|
+
______________________________________________________________________
|
|
502
|
+
|
|
503
|
+
## Development
|
|
504
|
+
|
|
505
|
+
Contributor workflow: [CONTRIBUTING.md](./CONTRIBUTING.md). Maintainer/agent notes: [CLAUDE.md](./CLAUDE.md).
|
|
506
|
+
|
|
507
|
+
**Setup:**
|
|
508
|
+
|
|
509
|
+
```bash
|
|
510
|
+
uv sync --extra dev
|
|
511
|
+
uvx pre-commit install
|
|
512
|
+
```
|
|
513
|
+
|
|
514
|
+
**Tests:**
|
|
515
|
+
|
|
516
|
+
```bash
|
|
517
|
+
pytest -m "not integration" # unit only (102 tests)
|
|
518
|
+
pytest # with integration (requires Node.js + npx)
|
|
519
|
+
```
|
|
520
|
+
|
|
521
|
+
**Skill sync + plugin metadata:**
|
|
522
|
+
|
|
523
|
+
```bash
|
|
524
|
+
python scripts/validate_skill_sync.py
|
|
525
|
+
python scripts/validate_plugin_metadata.py
|
|
526
|
+
```
|
|
527
|
+
|
|
528
|
+
______________________________________________________________________
|
|
529
|
+
|
|
530
|
+
## Links
|
|
531
|
+
|
|
532
|
+
- [INSTALL.md](./INSTALL.md) — full install matrix, all targets, per-agent detail
|
|
533
|
+
- [CONTRIBUTING.md](./CONTRIBUTING.md) — contributor workflow
|
|
534
|
+
- [CLAUDE.md](./CLAUDE.md) — maintainer guide for agents
|
|
535
|
+
- [CHANGELOG.md](./CHANGELOG.md) — release history
|
|
536
|
+
- [SECURITY.md](./SECURITY.md) — vulnerability reporting
|
|
537
|
+
- [Live docs](https://andrii-su.github.io/datoon/) — `docs/`
|
|
538
|
+
- [Issues](https://github.com/andrii-su/datoon/issues) — bugs, features, questions
|
|
539
|
+
|
|
540
|
+
______________________________________________________________________
|
|
541
|
+
|
|
542
|
+
## License
|
|
543
|
+
|
|
544
|
+
MIT
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
datoon/__init__.py,sha256=OavePq41iONob_AVZTT22GDnlYrLJHc16z01vVBHrNc,713
|
|
2
|
+
datoon/_version.py,sha256=pvN114lCpRNcdz9ufJCnl1MMqgOtZU1RmZfWI_uxL9s,520
|
|
3
|
+
datoon/analyzer.py,sha256=NL0rC_XmJu0WLi2bn96ToDwIkW5egAsOxtBJ12otkFE,2728
|
|
4
|
+
datoon/cli.py,sha256=itod-jKwc4Zv0KybGJME_wR6W-FpMq7YJKl38Dcrg80,6336
|
|
5
|
+
datoon/converter.py,sha256=Yl4nQ6V8Zm1Ez78IfyHPnLlVieVeiwuP8UUBJRbAIaA,5093
|
|
6
|
+
datoon/mcp_server.py,sha256=X2ESgMhBnJbSzzKc782EWjVfKu4lLZZsKk2HqEAApQw,4055
|
|
7
|
+
datoon/models.py,sha256=XUBNBgaimhayPPUBXBn9__4atO4aYOslnXeK4leZnYY,1899
|
|
8
|
+
datoon/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
9
|
+
datoon/readers/__init__.py,sha256=bGLsmZkDcEOfv0MaG6nfhFrIZe2fEsP8CtH2MYzp1JU,2852
|
|
10
|
+
datoon/readers/columnar.py,sha256=cTEWcVKPU8_X2wiRVVbENQx1rJ376oVuPddBPev6t_A,1325
|
|
11
|
+
datoon/readers/csv.py,sha256=o5H4_7S1yWQJN6EV1sK4WZoCQlepaIHpIYj7qAnJnz4,862
|
|
12
|
+
datoon/readers/excel.py,sha256=7js86H4Uek23-sdymDiQizzvJOdgWb2KMboUvkTmm-Q,939
|
|
13
|
+
datoon/readers/jsonl.py,sha256=17R3t4gRP8W8RMhBSMMJ8oYfsHIlYNZ-g3AIKwmqL9c,806
|
|
14
|
+
datoon/readers/numbers.py,sha256=XcjJCm-lETX-KAYspSy5Z0rVMorDQcTYkAp8UJBy6fs,941
|
|
15
|
+
datoon/readers/xml.py,sha256=n7RxhN4RQvgqRTWSb2cntt5yx-WZEHhbkGjBWwxg0tc,1706
|
|
16
|
+
datoon/readers/yaml.py,sha256=I2Qn3thwp46Qn1NrAueJyTj6HNKmCNfOI5YWx4xEee4,874
|
|
17
|
+
datoon-1.7.0.dist-info/licenses/LICENSE,sha256=Ry7TWMRxA9Ll1YnX9v82SM0171Wywfz_lfw48ikCbc4,1071
|
|
18
|
+
datoon-1.7.0.dist-info/METADATA,sha256=PG9Ebf3pvR3Uwx6dbsWrnBfCXQujKfWIiq-RJeiTaUE,17483
|
|
19
|
+
datoon-1.7.0.dist-info/WHEEL,sha256=aeYiig01lYGDzBgS8HxWXOg3uV61G9ijOsup-k9o1sk,91
|
|
20
|
+
datoon-1.7.0.dist-info/entry_points.txt,sha256=oPUvtVJDeElFpOvwIfruOtyiDKzvucR77uotIBdYu8g,79
|
|
21
|
+
datoon-1.7.0.dist-info/scm_file_list.json,sha256=PJdbYcLxdX-WfpJTU02rhEejEhA48DiwTdWVNU-Imjg,4373
|
|
22
|
+
datoon-1.7.0.dist-info/scm_version.json,sha256=DWd-gdgHTCyLD8w3vy70Vs3DUrkihjgThvku0NY6Iec,160
|
|
23
|
+
datoon-1.7.0.dist-info/top_level.txt,sha256=XzHiXRJ7nHbP4PIqnUZt8vB4zEJ7sXKAGu0f12cTlgc,7
|
|
24
|
+
datoon-1.7.0.dist-info/RECORD,,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Andrii Suruhov
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|