trimitdown 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- trimitdown-0.1.0/LICENSE +21 -0
- trimitdown-0.1.0/MANIFEST.in +6 -0
- trimitdown-0.1.0/PKG-INFO +243 -0
- trimitdown-0.1.0/README.md +208 -0
- trimitdown-0.1.0/pyproject.toml +73 -0
- trimitdown-0.1.0/setup.cfg +4 -0
- trimitdown-0.1.0/src/trimitdown/__init__.py +12 -0
- trimitdown-0.1.0/src/trimitdown/cli.py +84 -0
- trimitdown-0.1.0/src/trimitdown/convert.py +170 -0
- trimitdown-0.1.0/src/trimitdown/tiktoken_cache/9b5ad71b2ce5302211f9c61530b329a4922fc6a4 +100256 -0
- trimitdown-0.1.0/src/trimitdown/tiktoken_cache/README.md +14 -0
- trimitdown-0.1.0/src/trimitdown.egg-info/PKG-INFO +243 -0
- trimitdown-0.1.0/src/trimitdown.egg-info/SOURCES.txt +15 -0
- trimitdown-0.1.0/src/trimitdown.egg-info/dependency_links.txt +1 -0
- trimitdown-0.1.0/src/trimitdown.egg-info/entry_points.txt +2 -0
- trimitdown-0.1.0/src/trimitdown.egg-info/requires.txt +10 -0
- trimitdown-0.1.0/src/trimitdown.egg-info/top_level.txt +1 -0
trimitdown-0.1.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Sergey Dreizin
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
# В sdist приложения setuptools по умолчанию кладёт всё дерево tests/, причём
|
|
2
|
+
# неполным: conftest.py и pdf_fixtures.py туда не попадают, поэтому запустить эти
|
|
3
|
+
# тесты у потребителя всё равно нельзя. Публикуемый архив -- не место для лишней
|
|
4
|
+
# поверхности, тем более в проекте, который однажды уже опубликовал в тестовых
|
|
5
|
+
# данных то, чего публиковать не следовало.
|
|
6
|
+
prune tests
|
|
@@ -0,0 +1,243 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: trimitdown
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Convert documents to LLM-ready markdown. Ruled-table-aware PDF extraction, offline, no cloud.
|
|
5
|
+
Author: Sergey Dreizin
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/serjdrej/trimitdown
|
|
8
|
+
Project-URL: Repository, https://github.com/serjdrej/trimitdown
|
|
9
|
+
Project-URL: Issues, https://github.com/serjdrej/trimitdown/issues
|
|
10
|
+
Keywords: markdown,pdf,llm,document-conversion,cli
|
|
11
|
+
Classifier: Development Status :: 4 - Beta
|
|
12
|
+
Classifier: Environment :: Console
|
|
13
|
+
Classifier: Intended Audience :: Developers
|
|
14
|
+
Classifier: Operating System :: OS Independent
|
|
15
|
+
Classifier: Programming Language :: Python :: 3
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
20
|
+
Classifier: Topic :: Text Processing :: Markup :: Markdown
|
|
21
|
+
Classifier: Topic :: Utilities
|
|
22
|
+
Requires-Python: >=3.10
|
|
23
|
+
Description-Content-Type: text/markdown
|
|
24
|
+
License-File: LICENSE
|
|
25
|
+
Requires-Dist: trimitdown-pdf==0.1.0
|
|
26
|
+
Requires-Dist: markitdown[docx,outlook,pdf,pptx,xls,xlsx]
|
|
27
|
+
Requires-Dist: tiktoken
|
|
28
|
+
Requires-Dist: python-pptx
|
|
29
|
+
Requires-Dist: pdfminer.six
|
|
30
|
+
Provides-Extra: server
|
|
31
|
+
Requires-Dist: fastapi; extra == "server"
|
|
32
|
+
Requires-Dist: uvicorn; extra == "server"
|
|
33
|
+
Requires-Dist: python-multipart; extra == "server"
|
|
34
|
+
Dynamic: license-file
|
|
35
|
+
|
|
36
|
+
<div align="center">
|
|
37
|
+
|
|
38
|
+
<picture>
|
|
39
|
+
<source media="(prefers-color-scheme: dark)" srcset="brand/wordmark_dark.svg">
|
|
40
|
+
<img src="brand/wordmark_light.svg" alt="TrimItDown" width="360">
|
|
41
|
+
</picture>
|
|
42
|
+
|
|
43
|
+
**Any document → clean, LLM-ready Markdown. On your own server, on every device you own.**
|
|
44
|
+
|
|
45
|
+
*Read this in: English (this file) · [русский](README.ru.md)*
|
|
46
|
+
|
|
47
|
+
[](https://github.com/serjdrej/trimitdown/releases/latest)
|
|
48
|
+
[](https://github.com/serjdrej/trimitdown/releases)
|
|
49
|
+
[](https://github.com/serjdrej/trimitdown/actions/workflows/build-macos.yml)
|
|
50
|
+
[](https://pypi.org/project/trimitdown-pdf/)
|
|
51
|
+
[](LICENSE)
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
<picture>
|
|
55
|
+
<source media="(prefers-color-scheme: dark)" srcset="docs/images/result-dark.png">
|
|
56
|
+
<img src="docs/images/result-light.png" alt="TrimItDown converting a PDF: live Markdown preview with a real table and a token counter" width="850">
|
|
57
|
+
</picture>
|
|
58
|
+
|
|
59
|
+
</div>
|
|
60
|
+
|
|
61
|
+
TrimItDown turns PDF, Word (docx), PowerPoint (pptx), Excel (xlsx/xls) and Outlook (.msg)
|
|
62
|
+
files into clean, readable Markdown — ready to paste into Claude/ChatGPT, Obsidian, Notion,
|
|
63
|
+
or any markdown vault. It runs as an iPhone/iPad app (installed straight from Safari, no App
|
|
64
|
+
Store), a portable Windows program, a macOS app, and a self-hosted Docker server — with one
|
|
65
|
+
shared archive of conversions across all your devices.
|
|
66
|
+
|
|
67
|
+
## The PDF engine
|
|
68
|
+
|
|
69
|
+
Most converters — including the stock [MarkItDown](https://github.com/microsoft/markitdown)
|
|
70
|
+
PDF path — stumble on real-world PDFs in three measurable ways: they **glue words together**,
|
|
71
|
+
**invent tables out of ordinary prose**, and **drop genuine ruled tables**. Measured over 891
|
|
72
|
+
real documents from two independent collections, the stock converter emitted 5,624 table rows
|
|
73
|
+
on documents containing no ruled grid at all — prose reshaped into tables — against 2 from the
|
|
74
|
+
engine below. On the 885 of those documents that are not broken at the source, it roughly
|
|
75
|
+
halves glued words (107 runs vs 53). Those corpora are third-party
|
|
76
|
+
copyrighted material and are not redistributable — the full method, the numbers, the cases
|
|
77
|
+
where this engine *loses*, and what a reader **can** reproduce are in
|
|
78
|
+
[docs/pdf-engine.md](docs/pdf-engine.md).
|
|
79
|
+
|
|
80
|
+
So TrimItDown replaced it with a custom extraction engine:
|
|
81
|
+
|
|
82
|
+
- **Words split at a measured word-gap threshold** — a fraction of the font size, not a fixed
|
|
83
|
+
point value, so it holds across small print and large headings alike.
|
|
84
|
+
- **A dedicated table-detection stage** validates every candidate grid by how its cells are
|
|
85
|
+
actually filled (a row-fill vote), instead of trusting every ruled rectangle. Diagrams and
|
|
86
|
+
decorative frames get rejected; their text flows back into prose instead of vanishing.
|
|
87
|
+
- **Genuine ruled tables render as honest Markdown tables**, cell for cell.
|
|
88
|
+
|
|
89
|
+
Here is one document converted both ways. The file is committed to this repo, so you can run
|
|
90
|
+
the comparison yourself:
|
|
91
|
+
|
|
92
|
+
```bash
|
|
93
|
+
python scripts/compare_pdf_engines.py tests/data/sample-service-report.pdf
|
|
94
|
+
```
|
|
95
|
+
|
|
96
|
+
**Stock converter** — a phantom empty column splits the data, shifting every value one column
|
|
97
|
+
right of its header:
|
|
98
|
+
|
|
99
|
+
```markdown
|
|
100
|
+
| Parameter | Unit | Before service | | After service | Limit |
|
|
101
|
+
| ------------------------- | ---- | -------------- | --- | ------------- | ----- |
|
|
102
|
+
| Supply airflow | m³/h | | 352 | 398 | ≥ 380 |
|
|
103
|
+
| Extract airflow | m³/h | | 341 | 402 | ≥ 380 |
|
|
104
|
+
| Filter pressure drop | Pa | | 184 | 92 | ≤ 150 |
|
|
105
|
+
| Heat recovery efficiency | % | | 61 | 78 | ≥ 70 |
|
|
106
|
+
| Motor current, supply fan | A | | 1.9 | 1.6 | ≤ 2.2 |
|
|
107
|
+
```
|
|
108
|
+
|
|
109
|
+
**TrimItDown** — the table as it appears on the page:
|
|
110
|
+
|
|
111
|
+
```markdown
|
|
112
|
+
| Parameter | Unit | Before service | After service | Limit |
|
|
113
|
+
| --- | --- | --- | --- | --- |
|
|
114
|
+
| Supply airflow | m³/h | 352 | 398 | ≥ 380 |
|
|
115
|
+
| Extract airflow | m³/h | 341 | 402 | ≥ 380 |
|
|
116
|
+
| Filter pressure drop | Pa | 184 | 92 | ≤ 150 |
|
|
117
|
+
| Heat recovery efficiency | % | 61 | 78 | ≥ 70 |
|
|
118
|
+
| Motor current, supply fan | A | 1.9 | 1.6 | ≤ 2.2 |
|
|
119
|
+
```
|
|
120
|
+
|
|
121
|
+
One sample proves nothing about a corpus, and ours is third-party material we cannot publish.
|
|
122
|
+
So the measurement ships instead of the documents — point it at your own PDFs and it runs both
|
|
123
|
+
converters over all of them and prints a comparable summary, counts only, no filenames:
|
|
124
|
+
|
|
125
|
+
```bash
|
|
126
|
+
python scripts/measure_corpus.py /path/to/your/pdfs --limit 50
|
|
127
|
+
```
|
|
128
|
+
|
|
129
|
+
**Please send that summary back whatever it says** — a win, a loss, or a tie
|
|
130
|
+
([issue template](https://github.com/serjdrej/trimitdown/issues/new?template=measurement.yml)).
|
|
131
|
+
Results collected only when something breaks are a sample made of breakage, and nothing
|
|
132
|
+
honest can be concluded from it. Pooled across enough corpora, these numbers stop being one
|
|
133
|
+
person's collection. Details in
|
|
134
|
+
[docs/pdf-engine.md](docs/pdf-engine.md#re-measuring-on-your-own-pdfs).
|
|
135
|
+
|
|
136
|
+
In short: we add the table-*validation* stage that the classic tabula-java pipeline has and
|
|
137
|
+
Python extractors lack, expressed as a cell-fill vote on pdfplumber's ruled grids, with
|
|
138
|
+
per-grid fallback to prose — no ML models, no cloud, small enough to ship inside a portable
|
|
139
|
+
binary. Every non-PDF format still goes through MarkItDown.
|
|
140
|
+
|
|
141
|
+
## Why TrimItDown
|
|
142
|
+
|
|
143
|
+
- **Made for the LLM workflow.** Clean Markdown out, a live preview, and a token counter that
|
|
144
|
+
shows what a document will cost before you paste it into a model's context.
|
|
145
|
+
- **Your files never leave your infrastructure.** Conversion happens on your own server (home
|
|
146
|
+
NAS, VPS) or fully offline on your computer. No third-party SaaS, no per-page fees.
|
|
147
|
+
- **One archive, every device.** Convert on your phone — the result is already on your
|
|
148
|
+
computer, and vice versa. Searchable, with batch conversion and ZIP export.
|
|
149
|
+
- **A real app experience everywhere.** iPhone PWA installed from Safari (no App Store), a
|
|
150
|
+
portable Windows exe, a macOS app. Russian and English UI, light/dark, two color themes.
|
|
151
|
+
|
|
152
|
+
## Get it
|
|
153
|
+
|
|
154
|
+
| Platform | How |
|
|
155
|
+
|---|---|
|
|
156
|
+
| **Windows** | Download `TrimItDown-windows-x64.exe` from [Releases](https://github.com/serjdrej/trimitdown/releases/latest) — portable, no install |
|
|
157
|
+
| **macOS** (Apple Silicon / Intel) | Download the matching `.zip` from [Releases](https://github.com/serjdrej/trimitdown/releases/latest), unzip, right-click → Open |
|
|
158
|
+
| **iPhone / iPad** | Served by your own Docker server — open it in Safari → Share → *Add to Home Screen* |
|
|
159
|
+
| **Docker server** | See [Self-hosting](#self-hosting) below |
|
|
160
|
+
|
|
161
|
+
The desktop apps work fully offline out of the box. Point them at your server in **Settings**
|
|
162
|
+
to get the shared archive.
|
|
163
|
+
|
|
164
|
+
## Command-line package
|
|
165
|
+
|
|
166
|
+
Run a conversion without installing anything permanently:
|
|
167
|
+
|
|
168
|
+
```bash
|
|
169
|
+
uvx trimitdown convert report.pdf > report.md
|
|
170
|
+
```
|
|
171
|
+
|
|
172
|
+
Or install it with `pip install trimitdown` and write directly to a file with
|
|
173
|
+
`trimitdown convert report.pdf -o report.md`. Without `-o`, Markdown goes to standard output.
|
|
174
|
+
The CLI works fully offline.
|
|
175
|
+
|
|
176
|
+
## Self-hosting
|
|
177
|
+
|
|
178
|
+
The Docker server is the source of truth: it converts, stores the shared archive, and serves
|
|
179
|
+
the iPhone PWA.
|
|
180
|
+
|
|
181
|
+
```bash
|
|
182
|
+
git clone https://github.com/serjdrej/trimitdown.git
|
|
183
|
+
cd trimitdown/docker-server
|
|
184
|
+
# one-time: generate a self-signed HTTPS certificate (copy-paste command in docker-server/README.md)
|
|
185
|
+
docker-compose up -d --build
|
|
186
|
+
```
|
|
187
|
+
|
|
188
|
+
Then open `https://YOUR_SERVER:8002`. Full instructions — certificate generation, trusting it
|
|
189
|
+
on iOS/Windows/macOS, and the API — in [`docker-server/README.en.md`](docker-server/README.en.md).
|
|
190
|
+
|
|
191
|
+
## Screenshots
|
|
192
|
+
|
|
193
|
+
| Archive, shared across devices | Settings |
|
|
194
|
+
|---|---|
|
|
195
|
+
|  |  |
|
|
196
|
+
|
|
197
|
+
| Ocean theme, light | Ocean theme, dark |
|
|
198
|
+
|---|---|
|
|
199
|
+
|  |  |
|
|
200
|
+
|
|
201
|
+
| The iPhone PWA — home screen | Converting a PDF |
|
|
202
|
+
|---|---|
|
|
203
|
+
| <img src="docs/images/iphone-home-en.png" alt="TrimItDown PWA on iPhone, empty state" width="260"> | <img src="docs/images/iphone-result-en.png" alt="TrimItDown PWA on iPhone, conversion result" width="260"> |
|
|
204
|
+
|
|
205
|
+
## How it works
|
|
206
|
+
|
|
207
|
+
- **The Docker server** is a FastAPI service with HTTPS; the archive lives on the server. The
|
|
208
|
+
same service serves the PWA for iPhone.
|
|
209
|
+
- **The Windows/macOS apps** check at startup whether your server is reachable: if yes, they
|
|
210
|
+
open straight on it (shared archive); if not, they spin up a bundled local server and work
|
|
211
|
+
fully offline. The shell is [pywebview](https://pywebview.flowrl.com/) (WebView2 / WKWebView),
|
|
212
|
+
packaged with PyInstaller.
|
|
213
|
+
- **Conversion:** PDFs go through TrimItDown's own engine ([`trimitdown-pdf`](packages/trimitdown-pdf),
|
|
214
|
+
built on pdfplumber); every other format goes through Microsoft's
|
|
215
|
+
[MarkItDown](https://github.com/microsoft/markitdown).
|
|
216
|
+
|
|
217
|
+
## Repository layout
|
|
218
|
+
|
|
219
|
+
- [`src/trimitdown/`](src/trimitdown/) — published CLI package and conversion logic
|
|
220
|
+
- [`core/`](core/) — HTTP-layer and archive support for the desktop and server applications
|
|
221
|
+
- [`packages/trimitdown-pdf/`](packages/trimitdown-pdf/) — separately published PDF engine
|
|
222
|
+
- [`docker-server/`](docker-server/) — the self-hosted service + iPhone PWA
|
|
223
|
+
- [`static/`](static/), [`main.py`](main.py), [`server_app.py`](server_app.py) — desktop apps (UI, entry point, offline mode)
|
|
224
|
+
- [`tests/`](tests/) — unit tests + a labeled corpus harness for the table-detection stage
|
|
225
|
+
- [`docs/pdf-engine.md`](docs/pdf-engine.md) — PDF engine design, measurements, reproduction
|
|
226
|
+
|
|
227
|
+
## Contributing
|
|
228
|
+
|
|
229
|
+
Bug reports and PRs are welcome — see [CONTRIBUTING.md](CONTRIBUTING.md).
|
|
230
|
+
[DEVELOPMENT.md](DEVELOPMENT.md) documents the dev setup and the desktop build internals.
|
|
231
|
+
|
|
232
|
+
## Limitations
|
|
233
|
+
|
|
234
|
+
- The server's self-signed HTTPS certificate needs a one-time manual trust on each device
|
|
235
|
+
(iOS: profile + full trust; Windows: import into `CurrentUser\Root`; macOS: Keychain).
|
|
236
|
+
- The binaries aren't code-signed (no Apple/Microsoft developer certificate) — first launch
|
|
237
|
+
needs a one-time override (macOS: right-click → Open; Windows: SmartScreen).
|
|
238
|
+
|
|
239
|
+
## License and credits
|
|
240
|
+
|
|
241
|
+
Code is [MIT](LICENSE). Non-PDF conversion is powered by
|
|
242
|
+
[MarkItDown](https://github.com/microsoft/markitdown) (Microsoft, MIT); the bundled
|
|
243
|
+
third-party licenses are listed in [THIRD_PARTY_NOTICES.md](THIRD_PARTY_NOTICES.md).
|
|
@@ -0,0 +1,208 @@
|
|
|
1
|
+
<div align="center">
|
|
2
|
+
|
|
3
|
+
<picture>
|
|
4
|
+
<source media="(prefers-color-scheme: dark)" srcset="brand/wordmark_dark.svg">
|
|
5
|
+
<img src="brand/wordmark_light.svg" alt="TrimItDown" width="360">
|
|
6
|
+
</picture>
|
|
7
|
+
|
|
8
|
+
**Any document → clean, LLM-ready Markdown. On your own server, on every device you own.**
|
|
9
|
+
|
|
10
|
+
*Read this in: English (this file) · [русский](README.ru.md)*
|
|
11
|
+
|
|
12
|
+
[](https://github.com/serjdrej/trimitdown/releases/latest)
|
|
13
|
+
[](https://github.com/serjdrej/trimitdown/releases)
|
|
14
|
+
[](https://github.com/serjdrej/trimitdown/actions/workflows/build-macos.yml)
|
|
15
|
+
[](https://pypi.org/project/trimitdown-pdf/)
|
|
16
|
+
[](LICENSE)
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
<picture>
|
|
20
|
+
<source media="(prefers-color-scheme: dark)" srcset="docs/images/result-dark.png">
|
|
21
|
+
<img src="docs/images/result-light.png" alt="TrimItDown converting a PDF: live Markdown preview with a real table and a token counter" width="850">
|
|
22
|
+
</picture>
|
|
23
|
+
|
|
24
|
+
</div>
|
|
25
|
+
|
|
26
|
+
TrimItDown turns PDF, Word (docx), PowerPoint (pptx), Excel (xlsx/xls) and Outlook (.msg)
|
|
27
|
+
files into clean, readable Markdown — ready to paste into Claude/ChatGPT, Obsidian, Notion,
|
|
28
|
+
or any markdown vault. It runs as an iPhone/iPad app (installed straight from Safari, no App
|
|
29
|
+
Store), a portable Windows program, a macOS app, and a self-hosted Docker server — with one
|
|
30
|
+
shared archive of conversions across all your devices.
|
|
31
|
+
|
|
32
|
+
## The PDF engine
|
|
33
|
+
|
|
34
|
+
Most converters — including the stock [MarkItDown](https://github.com/microsoft/markitdown)
|
|
35
|
+
PDF path — stumble on real-world PDFs in three measurable ways: they **glue words together**,
|
|
36
|
+
**invent tables out of ordinary prose**, and **drop genuine ruled tables**. Measured over 891
|
|
37
|
+
real documents from two independent collections, the stock converter emitted 5,624 table rows
|
|
38
|
+
on documents containing no ruled grid at all — prose reshaped into tables — against 2 from the
|
|
39
|
+
engine below. On the 885 of those documents that are not broken at the source, it roughly
|
|
40
|
+
halves glued words (107 runs vs 53). Those corpora are third-party
|
|
41
|
+
copyrighted material and are not redistributable — the full method, the numbers, the cases
|
|
42
|
+
where this engine *loses*, and what a reader **can** reproduce are in
|
|
43
|
+
[docs/pdf-engine.md](docs/pdf-engine.md).
|
|
44
|
+
|
|
45
|
+
So TrimItDown replaced it with a custom extraction engine:
|
|
46
|
+
|
|
47
|
+
- **Words split at a measured word-gap threshold** — a fraction of the font size, not a fixed
|
|
48
|
+
point value, so it holds across small print and large headings alike.
|
|
49
|
+
- **A dedicated table-detection stage** validates every candidate grid by how its cells are
|
|
50
|
+
actually filled (a row-fill vote), instead of trusting every ruled rectangle. Diagrams and
|
|
51
|
+
decorative frames get rejected; their text flows back into prose instead of vanishing.
|
|
52
|
+
- **Genuine ruled tables render as honest Markdown tables**, cell for cell.
|
|
53
|
+
|
|
54
|
+
Here is one document converted both ways. The file is committed to this repo, so you can run
|
|
55
|
+
the comparison yourself:
|
|
56
|
+
|
|
57
|
+
```bash
|
|
58
|
+
python scripts/compare_pdf_engines.py tests/data/sample-service-report.pdf
|
|
59
|
+
```
|
|
60
|
+
|
|
61
|
+
**Stock converter** — a phantom empty column splits the data, shifting every value one column
|
|
62
|
+
right of its header:
|
|
63
|
+
|
|
64
|
+
```markdown
|
|
65
|
+
| Parameter | Unit | Before service | | After service | Limit |
|
|
66
|
+
| ------------------------- | ---- | -------------- | --- | ------------- | ----- |
|
|
67
|
+
| Supply airflow | m³/h | | 352 | 398 | ≥ 380 |
|
|
68
|
+
| Extract airflow | m³/h | | 341 | 402 | ≥ 380 |
|
|
69
|
+
| Filter pressure drop | Pa | | 184 | 92 | ≤ 150 |
|
|
70
|
+
| Heat recovery efficiency | % | | 61 | 78 | ≥ 70 |
|
|
71
|
+
| Motor current, supply fan | A | | 1.9 | 1.6 | ≤ 2.2 |
|
|
72
|
+
```
|
|
73
|
+
|
|
74
|
+
**TrimItDown** — the table as it appears on the page:
|
|
75
|
+
|
|
76
|
+
```markdown
|
|
77
|
+
| Parameter | Unit | Before service | After service | Limit |
|
|
78
|
+
| --- | --- | --- | --- | --- |
|
|
79
|
+
| Supply airflow | m³/h | 352 | 398 | ≥ 380 |
|
|
80
|
+
| Extract airflow | m³/h | 341 | 402 | ≥ 380 |
|
|
81
|
+
| Filter pressure drop | Pa | 184 | 92 | ≤ 150 |
|
|
82
|
+
| Heat recovery efficiency | % | 61 | 78 | ≥ 70 |
|
|
83
|
+
| Motor current, supply fan | A | 1.9 | 1.6 | ≤ 2.2 |
|
|
84
|
+
```
|
|
85
|
+
|
|
86
|
+
One sample proves nothing about a corpus, and ours is third-party material we cannot publish.
|
|
87
|
+
So the measurement ships instead of the documents — point it at your own PDFs and it runs both
|
|
88
|
+
converters over all of them and prints a comparable summary, counts only, no filenames:
|
|
89
|
+
|
|
90
|
+
```bash
|
|
91
|
+
python scripts/measure_corpus.py /path/to/your/pdfs --limit 50
|
|
92
|
+
```
|
|
93
|
+
|
|
94
|
+
**Please send that summary back whatever it says** — a win, a loss, or a tie
|
|
95
|
+
([issue template](https://github.com/serjdrej/trimitdown/issues/new?template=measurement.yml)).
|
|
96
|
+
Results collected only when something breaks are a sample made of breakage, and nothing
|
|
97
|
+
honest can be concluded from it. Pooled across enough corpora, these numbers stop being one
|
|
98
|
+
person's collection. Details in
|
|
99
|
+
[docs/pdf-engine.md](docs/pdf-engine.md#re-measuring-on-your-own-pdfs).
|
|
100
|
+
|
|
101
|
+
In short: we add the table-*validation* stage that the classic tabula-java pipeline has and
|
|
102
|
+
Python extractors lack, expressed as a cell-fill vote on pdfplumber's ruled grids, with
|
|
103
|
+
per-grid fallback to prose — no ML models, no cloud, small enough to ship inside a portable
|
|
104
|
+
binary. Every non-PDF format still goes through MarkItDown.
|
|
105
|
+
|
|
106
|
+
## Why TrimItDown
|
|
107
|
+
|
|
108
|
+
- **Made for the LLM workflow.** Clean Markdown out, a live preview, and a token counter that
|
|
109
|
+
shows what a document will cost before you paste it into a model's context.
|
|
110
|
+
- **Your files never leave your infrastructure.** Conversion happens on your own server (home
|
|
111
|
+
NAS, VPS) or fully offline on your computer. No third-party SaaS, no per-page fees.
|
|
112
|
+
- **One archive, every device.** Convert on your phone — the result is already on your
|
|
113
|
+
computer, and vice versa. Searchable, with batch conversion and ZIP export.
|
|
114
|
+
- **A real app experience everywhere.** iPhone PWA installed from Safari (no App Store), a
|
|
115
|
+
portable Windows exe, a macOS app. Russian and English UI, light/dark, two color themes.
|
|
116
|
+
|
|
117
|
+
## Get it
|
|
118
|
+
|
|
119
|
+
| Platform | How |
|
|
120
|
+
|---|---|
|
|
121
|
+
| **Windows** | Download `TrimItDown-windows-x64.exe` from [Releases](https://github.com/serjdrej/trimitdown/releases/latest) — portable, no install |
|
|
122
|
+
| **macOS** (Apple Silicon / Intel) | Download the matching `.zip` from [Releases](https://github.com/serjdrej/trimitdown/releases/latest), unzip, right-click → Open |
|
|
123
|
+
| **iPhone / iPad** | Served by your own Docker server — open it in Safari → Share → *Add to Home Screen* |
|
|
124
|
+
| **Docker server** | See [Self-hosting](#self-hosting) below |
|
|
125
|
+
|
|
126
|
+
The desktop apps work fully offline out of the box. Point them at your server in **Settings**
|
|
127
|
+
to get the shared archive.
|
|
128
|
+
|
|
129
|
+
## Command-line package
|
|
130
|
+
|
|
131
|
+
Run a conversion without installing anything permanently:
|
|
132
|
+
|
|
133
|
+
```bash
|
|
134
|
+
uvx trimitdown convert report.pdf > report.md
|
|
135
|
+
```
|
|
136
|
+
|
|
137
|
+
Or install it with `pip install trimitdown` and write directly to a file with
|
|
138
|
+
`trimitdown convert report.pdf -o report.md`. Without `-o`, Markdown goes to standard output.
|
|
139
|
+
The CLI works fully offline.
|
|
140
|
+
|
|
141
|
+
## Self-hosting
|
|
142
|
+
|
|
143
|
+
The Docker server is the source of truth: it converts, stores the shared archive, and serves
|
|
144
|
+
the iPhone PWA.
|
|
145
|
+
|
|
146
|
+
```bash
|
|
147
|
+
git clone https://github.com/serjdrej/trimitdown.git
|
|
148
|
+
cd trimitdown/docker-server
|
|
149
|
+
# one-time: generate a self-signed HTTPS certificate (copy-paste command in docker-server/README.md)
|
|
150
|
+
docker-compose up -d --build
|
|
151
|
+
```
|
|
152
|
+
|
|
153
|
+
Then open `https://YOUR_SERVER:8002`. Full instructions — certificate generation, trusting it
|
|
154
|
+
on iOS/Windows/macOS, and the API — in [`docker-server/README.en.md`](docker-server/README.en.md).
|
|
155
|
+
|
|
156
|
+
## Screenshots
|
|
157
|
+
|
|
158
|
+
| Archive, shared across devices | Settings |
|
|
159
|
+
|---|---|
|
|
160
|
+
|  |  |
|
|
161
|
+
|
|
162
|
+
| Ocean theme, light | Ocean theme, dark |
|
|
163
|
+
|---|---|
|
|
164
|
+
|  |  |
|
|
165
|
+
|
|
166
|
+
| The iPhone PWA — home screen | Converting a PDF |
|
|
167
|
+
|---|---|
|
|
168
|
+
| <img src="docs/images/iphone-home-en.png" alt="TrimItDown PWA on iPhone, empty state" width="260"> | <img src="docs/images/iphone-result-en.png" alt="TrimItDown PWA on iPhone, conversion result" width="260"> |
|
|
169
|
+
|
|
170
|
+
## How it works
|
|
171
|
+
|
|
172
|
+
- **The Docker server** is a FastAPI service with HTTPS; the archive lives on the server. The
|
|
173
|
+
same service serves the PWA for iPhone.
|
|
174
|
+
- **The Windows/macOS apps** check at startup whether your server is reachable: if yes, they
|
|
175
|
+
open straight on it (shared archive); if not, they spin up a bundled local server and work
|
|
176
|
+
fully offline. The shell is [pywebview](https://pywebview.flowrl.com/) (WebView2 / WKWebView),
|
|
177
|
+
packaged with PyInstaller.
|
|
178
|
+
- **Conversion:** PDFs go through TrimItDown's own engine ([`trimitdown-pdf`](packages/trimitdown-pdf),
|
|
179
|
+
built on pdfplumber); every other format goes through Microsoft's
|
|
180
|
+
[MarkItDown](https://github.com/microsoft/markitdown).
|
|
181
|
+
|
|
182
|
+
## Repository layout
|
|
183
|
+
|
|
184
|
+
- [`src/trimitdown/`](src/trimitdown/) — published CLI package and conversion logic
|
|
185
|
+
- [`core/`](core/) — HTTP-layer and archive support for the desktop and server applications
|
|
186
|
+
- [`packages/trimitdown-pdf/`](packages/trimitdown-pdf/) — separately published PDF engine
|
|
187
|
+
- [`docker-server/`](docker-server/) — the self-hosted service + iPhone PWA
|
|
188
|
+
- [`static/`](static/), [`main.py`](main.py), [`server_app.py`](server_app.py) — desktop apps (UI, entry point, offline mode)
|
|
189
|
+
- [`tests/`](tests/) — unit tests + a labeled corpus harness for the table-detection stage
|
|
190
|
+
- [`docs/pdf-engine.md`](docs/pdf-engine.md) — PDF engine design, measurements, reproduction
|
|
191
|
+
|
|
192
|
+
## Contributing
|
|
193
|
+
|
|
194
|
+
Bug reports and PRs are welcome — see [CONTRIBUTING.md](CONTRIBUTING.md).
|
|
195
|
+
[DEVELOPMENT.md](DEVELOPMENT.md) documents the dev setup and the desktop build internals.
|
|
196
|
+
|
|
197
|
+
## Limitations
|
|
198
|
+
|
|
199
|
+
- The server's self-signed HTTPS certificate needs a one-time manual trust on each device
|
|
200
|
+
(iOS: profile + full trust; Windows: import into `CurrentUser\Root`; macOS: Keychain).
|
|
201
|
+
- The binaries aren't code-signed (no Apple/Microsoft developer certificate) — first launch
|
|
202
|
+
needs a one-time override (macOS: right-click → Open; Windows: SmartScreen).
|
|
203
|
+
|
|
204
|
+
## License and credits
|
|
205
|
+
|
|
206
|
+
Code is [MIT](LICENSE). Non-PDF conversion is powered by
|
|
207
|
+
[MarkItDown](https://github.com/microsoft/markitdown) (Microsoft, MIT); the bundled
|
|
208
|
+
third-party licenses are listed in [THIRD_PARTY_NOTICES.md](THIRD_PARTY_NOTICES.md).
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=77"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "trimitdown"
|
|
7
|
+
# Единственный источник -- __version__ в модуле, читается статически через AST,
|
|
8
|
+
# без импорта: иначе в build-окружение потянулся бы markitdown.
|
|
9
|
+
dynamic = ["version"]
|
|
10
|
+
description = "Convert documents to LLM-ready markdown. Ruled-table-aware PDF extraction, offline, no cloud."
|
|
11
|
+
readme = "README.md"
|
|
12
|
+
requires-python = ">=3.10"
|
|
13
|
+
license = "MIT"
|
|
14
|
+
license-files = ["LICENSE"]
|
|
15
|
+
authors = [{ name = "Sergey Dreizin" }]
|
|
16
|
+
keywords = ["markdown", "pdf", "llm", "document-conversion", "cli"]
|
|
17
|
+
# Классификатор лицензии не добавлять: PEP 639 запрещает его вместе с
|
|
18
|
+
# выражением license = "MIT", setuptools>=77 падает.
|
|
19
|
+
classifiers = [
|
|
20
|
+
"Development Status :: 4 - Beta",
|
|
21
|
+
"Environment :: Console",
|
|
22
|
+
# Developers rather than desktop end users: what ships to PyPI is the CLI,
|
|
23
|
+
# and the desktop app never arrives through pip at all -- it comes via
|
|
24
|
+
# brew/scoop. The audience classifier naming desktop end users also happens
|
|
25
|
+
# to contain a substring the PII guard treats as a user path; see .githooks.
|
|
26
|
+
"Intended Audience :: Developers",
|
|
27
|
+
"Operating System :: OS Independent",
|
|
28
|
+
"Programming Language :: Python :: 3",
|
|
29
|
+
"Programming Language :: Python :: 3.10",
|
|
30
|
+
"Programming Language :: Python :: 3.11",
|
|
31
|
+
"Programming Language :: Python :: 3.12",
|
|
32
|
+
"Programming Language :: Python :: 3.13",
|
|
33
|
+
"Topic :: Text Processing :: Markup :: Markdown",
|
|
34
|
+
"Topic :: Utilities",
|
|
35
|
+
]
|
|
36
|
+
|
|
37
|
+
# Точный пин, не диапазон: диапазон дал бы молчаливый подъём движка при
|
|
38
|
+
# пересборке, а движок определяет качество вывода. Подъём -- отдельный коммит.
|
|
39
|
+
# FastAPI/uvicorn здесь НЕТ: это CLI-пакет, веб-стек живёт в extra "server".
|
|
40
|
+
dependencies = [
|
|
41
|
+
"trimitdown-pdf==0.1.0",
|
|
42
|
+
# Same narrow extras as requirements.txt, and for the same reason recorded
|
|
43
|
+
# there: the audio-transcription and Azure extras are heavy and useless
|
|
44
|
+
# offline, which is what this package claims to be. "all" would pull an
|
|
45
|
+
# Azure client into every install.
|
|
46
|
+
"markitdown[docx,pdf,pptx,xlsx,xls,outlook]",
|
|
47
|
+
"tiktoken",
|
|
48
|
+
"python-pptx",
|
|
49
|
+
"pdfminer.six",
|
|
50
|
+
]
|
|
51
|
+
|
|
52
|
+
[project.optional-dependencies]
|
|
53
|
+
# Нужен только тому, кто поднимает HTTP-слой (Docker-образ, десктопная сборка).
|
|
54
|
+
server = ["fastapi", "uvicorn", "python-multipart"]
|
|
55
|
+
|
|
56
|
+
[project.scripts]
|
|
57
|
+
trimitdown = "trimitdown.cli:main"
|
|
58
|
+
|
|
59
|
+
[project.urls]
|
|
60
|
+
Homepage = "https://github.com/serjdrej/trimitdown"
|
|
61
|
+
Repository = "https://github.com/serjdrej/trimitdown"
|
|
62
|
+
Issues = "https://github.com/serjdrej/trimitdown/issues"
|
|
63
|
+
|
|
64
|
+
[tool.setuptools.dynamic]
|
|
65
|
+
version = { attr = "trimitdown.__version__" }
|
|
66
|
+
|
|
67
|
+
[tool.setuptools.packages.find]
|
|
68
|
+
where = ["src"]
|
|
69
|
+
|
|
70
|
+
[tool.setuptools.package-data]
|
|
71
|
+
# Офлайн-кэш tiktoken. Без этой строки он не попадёт в wheel, и счёт токенов у
|
|
72
|
+
# установленного пакета молча уйдёт в сеть.
|
|
73
|
+
trimitdown = ["tiktoken_cache/*"]
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
"""TrimItDown: конвертация документов в markdown, пригодный для вставки в LLM.
|
|
2
|
+
|
|
3
|
+
ЕДИНСТВЕННЫЙ источник версии для всего проекта: пакета на PyPI, десктопного
|
|
4
|
+
приложения, ответа /api/mode и того, что видно в UI. Раньше версия приложения
|
|
5
|
+
жила отдельно в core/version.py -- два числа неизбежно разъезжаются, и разъезд
|
|
6
|
+
замечает пользователь, а не мы.
|
|
7
|
+
|
|
8
|
+
Обязан остаться строковым литералом: setuptools читает его статически через AST,
|
|
9
|
+
не импортируя модуль, -- иначе в build-окружение потянется markitdown.
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
__version__ = "0.1.0"
|
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
"""Командная строка: единственный интерфейс pip/uvx-пакета.
|
|
2
|
+
|
|
3
|
+
Только локальная конверсия. Работа с внешним сервером -- это канал Docker
|
|
4
|
+
(docker exec внутрь контейнера) и настройка в десктопном приложении; сетевого
|
|
5
|
+
режима здесь нет сознательно, чтобы у инструмента, который ценен локальностью,
|
|
6
|
+
не появилось сетевых отказов.
|
|
7
|
+
"""
|
|
8
|
+
import argparse
|
|
9
|
+
import sys
|
|
10
|
+
from pathlib import Path
|
|
11
|
+
|
|
12
|
+
from trimitdown import __version__
|
|
13
|
+
from trimitdown.convert import ConversionError, convert_bytes, convert_path
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def _build_parser() -> argparse.ArgumentParser:
|
|
17
|
+
parser = argparse.ArgumentParser(
|
|
18
|
+
prog="trimitdown",
|
|
19
|
+
description="Convert documents to LLM-ready markdown.",
|
|
20
|
+
)
|
|
21
|
+
parser.add_argument("--version", action="version", version=f"trimitdown {__version__}")
|
|
22
|
+
|
|
23
|
+
sub = parser.add_subparsers(dest="command", required=True)
|
|
24
|
+
convert = sub.add_parser("convert", help="convert one document to markdown")
|
|
25
|
+
convert.add_argument("path", help='file to convert, or "-" to read stdin')
|
|
26
|
+
convert.add_argument("-o", "--output", help="write markdown here instead of stdout")
|
|
27
|
+
convert.add_argument(
|
|
28
|
+
"--type",
|
|
29
|
+
dest="suffix",
|
|
30
|
+
help='file extension for stdin input, e.g. ".pdf" -- required with "-"',
|
|
31
|
+
)
|
|
32
|
+
return parser
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def _run_convert(args) -> int:
|
|
36
|
+
if args.path == "-":
|
|
37
|
+
if not args.suffix:
|
|
38
|
+
print(
|
|
39
|
+
'reading from stdin needs --type, e.g. --type .pdf: the extension '
|
|
40
|
+
"cannot be recovered from a stream",
|
|
41
|
+
file=sys.stderr,
|
|
42
|
+
)
|
|
43
|
+
return 2
|
|
44
|
+
try:
|
|
45
|
+
result = convert_bytes(sys.stdin.buffer.read(), args.suffix)
|
|
46
|
+
except ConversionError as e:
|
|
47
|
+
print(f"could not convert stdin: {e}", file=sys.stderr)
|
|
48
|
+
return 1
|
|
49
|
+
else:
|
|
50
|
+
path = Path(args.path)
|
|
51
|
+
if not path.is_file():
|
|
52
|
+
print(f"no such file: {path}", file=sys.stderr)
|
|
53
|
+
return 1
|
|
54
|
+
try:
|
|
55
|
+
result = convert_path(path)
|
|
56
|
+
except ConversionError as e:
|
|
57
|
+
print(f"could not convert {path}: {e}", file=sys.stderr)
|
|
58
|
+
return 1
|
|
59
|
+
|
|
60
|
+
if args.output:
|
|
61
|
+
Path(args.output).write_text(result.text, encoding="utf-8")
|
|
62
|
+
else:
|
|
63
|
+
# Явная запись в буфер: на Windows консольная кодировка по умолчанию не
|
|
64
|
+
# переваривает вывод конвертации (m³/h, ≥, кириллица) и печать падает.
|
|
65
|
+
sys.stdout.buffer.write(result.text.encode("utf-8"))
|
|
66
|
+
return 0
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def main(argv: list[str] | None = None) -> int:
|
|
70
|
+
# Converted documents and the paths they came from carry characters the console
|
|
71
|
+
# codepage cannot encode (m³/h, ≥, Cyrillic); on a Russian Windows the first one
|
|
72
|
+
# would abort the process. stdout is written as bytes below for the document
|
|
73
|
+
# body, but error messages go through the text layer and need this.
|
|
74
|
+
for stream in (sys.stdout, sys.stderr):
|
|
75
|
+
stream.reconfigure(encoding="utf-8", errors="replace")
|
|
76
|
+
|
|
77
|
+
args = _build_parser().parse_args(argv)
|
|
78
|
+
if args.command == "convert":
|
|
79
|
+
return _run_convert(args)
|
|
80
|
+
return 2
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
if __name__ == "__main__":
|
|
84
|
+
raise SystemExit(main())
|