litedoc-cli 3.1.1__tar.gz → 3.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- litedoc_cli-3.3.0/PKG-INFO +57 -0
- litedoc_cli-3.3.0/README.md +47 -0
- {litedoc_cli-3.1.1 → litedoc_cli-3.3.0}/litedoc_cli/__init__.py +1 -1
- litedoc_cli-3.3.0/litedoc_cli/animation.py +163 -0
- {litedoc_cli-3.1.1 → litedoc_cli-3.3.0}/litedoc_cli/app/index.html +1210 -191
- litedoc_cli-3.3.0/litedoc_cli/benchmark.py +55 -0
- litedoc_cli-3.3.0/litedoc_cli/cli.py +474 -0
- {litedoc_cli-3.1.1 → litedoc_cli-3.3.0}/litedoc_cli/engine.py +10 -3
- litedoc_cli-3.3.0/litedoc_cli.egg-info/PKG-INFO +57 -0
- {litedoc_cli-3.1.1 → litedoc_cli-3.3.0}/litedoc_cli.egg-info/SOURCES.txt +2 -0
- {litedoc_cli-3.1.1 → litedoc_cli-3.3.0}/pyproject.toml +3 -3
- litedoc_cli-3.1.1/PKG-INFO +0 -40
- litedoc_cli-3.1.1/README.md +0 -30
- litedoc_cli-3.1.1/litedoc_cli/cli.py +0 -211
- litedoc_cli-3.1.1/litedoc_cli.egg-info/PKG-INFO +0 -40
- {litedoc_cli-3.1.1 → litedoc_cli-3.3.0}/litedoc_cli/__main__.py +0 -0
- {litedoc_cli-3.1.1 → litedoc_cli-3.3.0}/litedoc_cli/ai.py +0 -0
- {litedoc_cli-3.1.1 → litedoc_cli-3.3.0}/litedoc_cli.egg-info/dependency_links.txt +0 -0
- {litedoc_cli-3.1.1 → litedoc_cli-3.3.0}/litedoc_cli.egg-info/entry_points.txt +0 -0
- {litedoc_cli-3.1.1 → litedoc_cli-3.3.0}/litedoc_cli.egg-info/requires.txt +0 -0
- {litedoc_cli-3.1.1 → litedoc_cli-3.3.0}/litedoc_cli.egg-info/top_level.txt +0 -0
- {litedoc_cli-3.1.1 → litedoc_cli-3.3.0}/setup.cfg +0 -0
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: litedoc-cli
|
|
3
|
+
Version: 3.3.0
|
|
4
|
+
Summary: LiteDoc in your terminal: deterministic PDF → Markdown with an optional bring-your-own-AI repair pass. Same engine as litedoc.xyz.
|
|
5
|
+
License-Expression: MIT
|
|
6
|
+
Requires-Python: >=3.10
|
|
7
|
+
Description-Content-Type: text/markdown
|
|
8
|
+
Requires-Dist: playwright>=1.40
|
|
9
|
+
Requires-Dist: httpx>=0.25
|
|
10
|
+
|
|
11
|
+
# litedoc-cli
|
|
12
|
+
|
|
13
|
+
The [litedoc.xyz](https://litedoc.xyz) extraction engine in your terminal: deterministic PDF → Markdown that never hallucinates, engineered for server pipelines, RAG ingestion, and local workflow automation. Includes an optional, triage-first AI repair pass you can point at **your own** model.
|
|
14
|
+
|
|
15
|
+
```bash
|
|
16
|
+
pip install litedoc-cli
|
|
17
|
+
playwright install chromium # one-time engine download
|
|
18
|
+
|
|
19
|
+
# Basic & Batch Conversion
|
|
20
|
+
litedoc convert paper.pdf # markdown to stdout
|
|
21
|
+
litedoc convert scans/*.pdf -o out/ --ocr # batch + OCR fallback
|
|
22
|
+
litedoc convert paper.pdf -o out/ --images out/images # extracted figures as JPEGs
|
|
23
|
+
|
|
24
|
+
# Precision & Quality Controls (New in v3.2.0!)
|
|
25
|
+
litedoc convert textbook.pdf --pages "1-5,12-15" -o out/ # slice exact page ranges
|
|
26
|
+
litedoc convert diagrams.pdf --img-res 600 -o out/ # set custom figure DPI (75,150,300,600)
|
|
27
|
+
litedoc convert batch/*.pdf --auto-resolve clean # set OCR conflict handling strategy
|
|
28
|
+
litedoc convert massive.pdf -o out/ --verbose # stream live diagnostic stage events
|
|
29
|
+
|
|
30
|
+
# Live Automation Daemons (Obsidian / RAG Dropzones)
|
|
31
|
+
litedoc convert ~/obsidian/inbox -o ~/obsidian/notes --watch --recursive
|
|
32
|
+
|
|
33
|
+
# AI Repair (Bring Your Own Model)
|
|
34
|
+
litedoc convert scan.pdf --ai-url http://localhost:11434 --ai-model llama3.1:8b
|
|
35
|
+
```
|
|
36
|
+
|
|
37
|
+
## Features & Philosophy
|
|
38
|
+
|
|
39
|
+
* **Identical output to the web app** — it drives the same single-file engine headlessly.
|
|
40
|
+
* **Deterministic by default** — no AI flag, no network: nothing leaves your machine.
|
|
41
|
+
* **Precision Extraction & Page Slicing** — `--pages` lets you extract targeted chapters or sections without paying compute or token overhead for entire volumes.
|
|
42
|
+
* **Live Folder Watcher & Daemon Mode** — drop `--watch` (`-w`) and `--recursive` (`-r`) onto an ingestion directory to run an automated, zero-dependency background RAG pipeline that instantly turns newly dropped PDFs into structured Markdown.
|
|
43
|
+
* **Automated QA & Auto-Resolve** — configure batch conflict handling (`--auto-resolve {render,gibberish,clean,skip}`) so batch scripts run unattended without manual verification breaks.
|
|
44
|
+
* **Headless Performance Benchmarking** — test processing speeds on your hardware anytime:
|
|
45
|
+
```bash
|
|
46
|
+
litedoc benchmark --iterations 3 --json
|
|
47
|
+
```
|
|
48
|
+
Reports worker spin-up latency, burst PPT (Pages Per Toast), and sustained PPM (Pages Per Minute).
|
|
49
|
+
* **Persistent Global Configuration** — customize your default CLI conversion parameters so you don't have to repeat flags across invocations:
|
|
50
|
+
```bash
|
|
51
|
+
litedoc config --set img_res=600 auto_resolve=render verbose=true
|
|
52
|
+
litedoc config # inspect currently stored defaults (~/.config/litedoc/config.json)
|
|
53
|
+
```
|
|
54
|
+
* **Bring your own AI** — `--ai-url` speaks Ollama or OpenAI-compatible protocols; only sections that are provably damaged (broken sentences, ragged tables, OCR artifacts) are ever sent. `--ai` uses the hosted LiteDoc service instead (token from `LITEDOC_TOKEN`).
|
|
55
|
+
* **Structure, not just prose** — `--json` returns per-page layout (lines with coordinates, font sizes, reading-order blocks, OCR provenance; tables as row arrays; figures with page + bounding box), and `--images DIR` writes detected figures — cropped to include their axis labels and annotations — as JPEG files linked from the markdown.
|
|
56
|
+
|
|
57
|
+
License: MIT. Part of the [LiteDoc](https://github.com/0xovo/LiteDoc) project.
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
# litedoc-cli
|
|
2
|
+
|
|
3
|
+
The [litedoc.xyz](https://litedoc.xyz) extraction engine in your terminal: deterministic PDF → Markdown that never hallucinates, engineered for server pipelines, RAG ingestion, and local workflow automation. Includes an optional, triage-first AI repair pass you can point at **your own** model.
|
|
4
|
+
|
|
5
|
+
```bash
|
|
6
|
+
pip install litedoc-cli
|
|
7
|
+
playwright install chromium # one-time engine download
|
|
8
|
+
|
|
9
|
+
# Basic & Batch Conversion
|
|
10
|
+
litedoc convert paper.pdf # markdown to stdout
|
|
11
|
+
litedoc convert scans/*.pdf -o out/ --ocr # batch + OCR fallback
|
|
12
|
+
litedoc convert paper.pdf -o out/ --images out/images # extracted figures as JPEGs
|
|
13
|
+
|
|
14
|
+
# Precision & Quality Controls (New in v3.2.0!)
|
|
15
|
+
litedoc convert textbook.pdf --pages "1-5,12-15" -o out/ # slice exact page ranges
|
|
16
|
+
litedoc convert diagrams.pdf --img-res 600 -o out/ # set custom figure DPI (75,150,300,600)
|
|
17
|
+
litedoc convert batch/*.pdf --auto-resolve clean # set OCR conflict handling strategy
|
|
18
|
+
litedoc convert massive.pdf -o out/ --verbose # stream live diagnostic stage events
|
|
19
|
+
|
|
20
|
+
# Live Automation Daemons (Obsidian / RAG Dropzones)
|
|
21
|
+
litedoc convert ~/obsidian/inbox -o ~/obsidian/notes --watch --recursive
|
|
22
|
+
|
|
23
|
+
# AI Repair (Bring Your Own Model)
|
|
24
|
+
litedoc convert scan.pdf --ai-url http://localhost:11434 --ai-model llama3.1:8b
|
|
25
|
+
```
|
|
26
|
+
|
|
27
|
+
## Features & Philosophy
|
|
28
|
+
|
|
29
|
+
* **Identical output to the web app** — it drives the same single-file engine headlessly.
|
|
30
|
+
* **Deterministic by default** — no AI flag, no network: nothing leaves your machine.
|
|
31
|
+
* **Precision Extraction & Page Slicing** — `--pages` lets you extract targeted chapters or sections without paying compute or token overhead for entire volumes.
|
|
32
|
+
* **Live Folder Watcher & Daemon Mode** — drop `--watch` (`-w`) and `--recursive` (`-r`) onto an ingestion directory to run an automated, zero-dependency background RAG pipeline that instantly turns newly dropped PDFs into structured Markdown.
|
|
33
|
+
* **Automated QA & Auto-Resolve** — configure batch conflict handling (`--auto-resolve {render,gibberish,clean,skip}`) so batch scripts run unattended without manual verification breaks.
|
|
34
|
+
* **Headless Performance Benchmarking** — test processing speeds on your hardware anytime:
|
|
35
|
+
```bash
|
|
36
|
+
litedoc benchmark --iterations 3 --json
|
|
37
|
+
```
|
|
38
|
+
Reports worker spin-up latency, burst PPT (Pages Per Toast), and sustained PPM (Pages Per Minute).
|
|
39
|
+
* **Persistent Global Configuration** — customize your default CLI conversion parameters so you don't have to repeat flags across invocations:
|
|
40
|
+
```bash
|
|
41
|
+
litedoc config --set img_res=600 auto_resolve=render verbose=true
|
|
42
|
+
litedoc config # inspect currently stored defaults (~/.config/litedoc/config.json)
|
|
43
|
+
```
|
|
44
|
+
* **Bring your own AI** — `--ai-url` speaks Ollama or OpenAI-compatible protocols; only sections that are provably damaged (broken sentences, ragged tables, OCR artifacts) are ever sent. `--ai` uses the hosted LiteDoc service instead (token from `LITEDOC_TOKEN`).
|
|
45
|
+
* **Structure, not just prose** — `--json` returns per-page layout (lines with coordinates, font sizes, reading-order blocks, OCR provenance; tables as row arrays; figures with page + bounding box), and `--images DIR` writes detected figures — cropped to include their axis labels and annotations — as JPEG files linked from the markdown.
|
|
46
|
+
|
|
47
|
+
License: MIT. Part of the [LiteDoc](https://github.com/0xovo/LiteDoc) project.
|
|
@@ -0,0 +1,163 @@
|
|
|
1
|
+
"""Animated ASCII startup sequence for LiteDoc CLI."""
|
|
2
|
+
|
|
3
|
+
import time
|
|
4
|
+
import sys
|
|
5
|
+
|
|
6
|
+
GRAY = "\033[90m"
|
|
7
|
+
MAGENTA = "\033[35m"
|
|
8
|
+
CYAN = "\033[36m"
|
|
9
|
+
RED = "\033[31m"
|
|
10
|
+
BOLD = "\033[1m"
|
|
11
|
+
RESET = "\033[0m"
|
|
12
|
+
|
|
13
|
+
frames = [
|
|
14
|
+
# Frame 0: Initial PDF State
|
|
15
|
+
[
|
|
16
|
+
f" {GRAY}┌───────────────┐{RESET}",
|
|
17
|
+
f" {GRAY}│{RESET} {RED}[PDF]{RESET} {GRAY}│{RESET}",
|
|
18
|
+
f" {GRAY}│{RESET} ≡≡≡≡≡≡≡≡≡≡≡≡≡ {GRAY}│{RESET}",
|
|
19
|
+
f" {GRAY}│{RESET} ≡≡≡≡≡≡≡ {GRAY}│{RESET}",
|
|
20
|
+
f" {GRAY}│{RESET} ≡≡≡≡≡≡≡≡≡≡≡≡≡ {GRAY}│{RESET}",
|
|
21
|
+
f" {GRAY}└───────────────┘{RESET}"
|
|
22
|
+
],
|
|
23
|
+
# Frame 1: Scanner moving down
|
|
24
|
+
[
|
|
25
|
+
f" {GRAY}┌───────────────┐{RESET}",
|
|
26
|
+
f" {MAGENTA}│ ============= │{RESET}",
|
|
27
|
+
f" {GRAY}│{RESET} ≡≡≡≡≡≡≡≡≡≡≡≡≡ {GRAY}│{RESET}",
|
|
28
|
+
f" {GRAY}│{RESET} ≡≡≡≡≡≡≡ {GRAY}│{RESET}",
|
|
29
|
+
f" {GRAY}│{RESET} ≡≡≡≡≡≡≡≡≡≡≡≡≡ {GRAY}│{RESET}",
|
|
30
|
+
f" {GRAY}└───────────────┘{RESET}"
|
|
31
|
+
],
|
|
32
|
+
# Frame 2: Markdown revealing
|
|
33
|
+
[
|
|
34
|
+
f" {GRAY}┌───────────────┐{RESET}",
|
|
35
|
+
f" {GRAY}│{RESET} {CYAN}# markdown{RESET} {GRAY}│{RESET}",
|
|
36
|
+
f" {MAGENTA}│ ============= │{RESET}",
|
|
37
|
+
f" {GRAY}│{RESET} ≡≡≡≡≡≡≡ {GRAY}│{RESET}",
|
|
38
|
+
f" {GRAY}│{RESET} ≡≡≡≡≡≡≡≡≡≡≡≡≡ {GRAY}│{RESET}",
|
|
39
|
+
f" {GRAY}└───────────────┘{RESET}"
|
|
40
|
+
],
|
|
41
|
+
# Frame 3: Markdown revealing
|
|
42
|
+
[
|
|
43
|
+
f" {GRAY}┌───────────────┐{RESET}",
|
|
44
|
+
f" {GRAY}│{RESET} {CYAN}# markdown{RESET} {GRAY}│{RESET}",
|
|
45
|
+
f" {GRAY}│{RESET} {CYAN}- paragraph{RESET} {GRAY}│{RESET}",
|
|
46
|
+
f" {MAGENTA}│ ============= │{RESET}",
|
|
47
|
+
f" {GRAY}│{RESET} ≡≡≡≡≡≡≡≡≡≡≡≡≡ {GRAY}│{RESET}",
|
|
48
|
+
f" {GRAY}└───────────────┘{RESET}"
|
|
49
|
+
],
|
|
50
|
+
# Frame 4: Markdown revealing
|
|
51
|
+
[
|
|
52
|
+
f" {GRAY}┌───────────────┐{RESET}",
|
|
53
|
+
f" {GRAY}│{RESET} {CYAN}# markdown{RESET} {GRAY}│{RESET}",
|
|
54
|
+
f" {GRAY}│{RESET} {CYAN}- paragraph{RESET} {GRAY}│{RESET}",
|
|
55
|
+
f" {GRAY}│{RESET} {CYAN}- bullet pt{RESET} {GRAY}│{RESET}",
|
|
56
|
+
f" {MAGENTA}│ ============= │{RESET}",
|
|
57
|
+
f" {GRAY}└───────────────┘{RESET}"
|
|
58
|
+
],
|
|
59
|
+
# Frame 5: Complete Markdown State
|
|
60
|
+
[
|
|
61
|
+
f" {GRAY}┌───────────────┐{RESET}",
|
|
62
|
+
f" {GRAY}│{RESET} {CYAN}# markdown{RESET} {GRAY}│{RESET}",
|
|
63
|
+
f" {GRAY}│{RESET} {CYAN}- paragraph{RESET} {GRAY}│{RESET}",
|
|
64
|
+
f" {GRAY}│{RESET} {CYAN}- bullet pt{RESET} {GRAY}│{RESET}",
|
|
65
|
+
f" {GRAY}│{RESET} {CYAN}> quote{RESET} {GRAY}│{RESET}",
|
|
66
|
+
f" {GRAY}└───────────────┘{RESET}"
|
|
67
|
+
],
|
|
68
|
+
# Frame 6: Shrinking layout
|
|
69
|
+
[
|
|
70
|
+
f"",
|
|
71
|
+
f" {GRAY}┌─────────┐{RESET}",
|
|
72
|
+
f" {GRAY}│{RESET} {CYAN}......{RESET} {GRAY}│{RESET}",
|
|
73
|
+
f" {GRAY}│{RESET} {CYAN}...{RESET} {GRAY}│{RESET}",
|
|
74
|
+
f" {GRAY}└─────────┘{RESET}",
|
|
75
|
+
f""
|
|
76
|
+
],
|
|
77
|
+
# Frame 7: Morphing into the litedoc logo
|
|
78
|
+
[
|
|
79
|
+
f"",
|
|
80
|
+
f"",
|
|
81
|
+
f" {GRAY}({RESET} {BOLD}litedoc{MAGENTA}.xyz{RESET} {GRAY}){RESET}",
|
|
82
|
+
f"",
|
|
83
|
+
f"",
|
|
84
|
+
f""
|
|
85
|
+
],
|
|
86
|
+
# Frame 8: Fading out
|
|
87
|
+
[
|
|
88
|
+
f"",
|
|
89
|
+
f"",
|
|
90
|
+
f" {GRAY}( litedoc.xyz ){RESET}",
|
|
91
|
+
f"",
|
|
92
|
+
f"",
|
|
93
|
+
f""
|
|
94
|
+
]
|
|
95
|
+
]
|
|
96
|
+
|
|
97
|
+
def run_animation(show_prompt=True, stream=None):
|
|
98
|
+
if stream is None:
|
|
99
|
+
stream = sys.stdout
|
|
100
|
+
try:
|
|
101
|
+
# Hide cursor
|
|
102
|
+
stream.write('\033[?25l')
|
|
103
|
+
stream.flush()
|
|
104
|
+
|
|
105
|
+
# Add a newline buffer
|
|
106
|
+
stream.write('\n')
|
|
107
|
+
|
|
108
|
+
# Loop through frames
|
|
109
|
+
for i, frame in enumerate(frames):
|
|
110
|
+
# If not the first frame, move cursor up 6 lines
|
|
111
|
+
if i > 0:
|
|
112
|
+
stream.write(f'\033[{len(frame)}A')
|
|
113
|
+
|
|
114
|
+
# Draw the current frame line-by-line
|
|
115
|
+
for line in frame:
|
|
116
|
+
# \033[2K clears the current line entirely to prevent ghosting
|
|
117
|
+
stream.write(f'\033[2K{line}\n')
|
|
118
|
+
|
|
119
|
+
stream.flush()
|
|
120
|
+
|
|
121
|
+
# Dynamic timing to match the React CSS keyframes
|
|
122
|
+
if i == 0 or i == 5:
|
|
123
|
+
delay = 0.5 # Pause on PDF and MD states
|
|
124
|
+
elif i == len(frames) - 2:
|
|
125
|
+
delay = 0.6 # Pause on Logo
|
|
126
|
+
elif i == len(frames) - 1:
|
|
127
|
+
delay = 0.2 # Quick fade out
|
|
128
|
+
else:
|
|
129
|
+
delay = 0.15 # Fast scanning/morphing speed
|
|
130
|
+
|
|
131
|
+
time.sleep(delay)
|
|
132
|
+
|
|
133
|
+
# Move up and erase the 6 lines of the animation space
|
|
134
|
+
stream.write(f'\033[{len(frames[0])}A')
|
|
135
|
+
for _ in range(len(frames[0])):
|
|
136
|
+
stream.write('\033[2K\n')
|
|
137
|
+
# Reset cursor back to top
|
|
138
|
+
stream.write(f'\033[{len(frames[0])}A')
|
|
139
|
+
|
|
140
|
+
# Move back up one line to eat the initial buffer
|
|
141
|
+
stream.write('\033[1A\033[2K')
|
|
142
|
+
|
|
143
|
+
# Simulated CLI App Start
|
|
144
|
+
stream.write(f"\n{MAGENTA}◆{RESET} {BOLD}litedoc.xyz{RESET} Local Engine initialized.\n")
|
|
145
|
+
if show_prompt:
|
|
146
|
+
stream.write(f"{GRAY}Ready to extract files.{RESET}\n\n")
|
|
147
|
+
stream.write(f"{CYAN}>{RESET} _\n")
|
|
148
|
+
else:
|
|
149
|
+
stream.write(f"{GRAY}Ready to extract files.{RESET}\n")
|
|
150
|
+
stream.flush()
|
|
151
|
+
|
|
152
|
+
except KeyboardInterrupt:
|
|
153
|
+
pass
|
|
154
|
+
finally:
|
|
155
|
+
# Always ensure the cursor is shown again before exiting
|
|
156
|
+
stream.write('\033[?25h')
|
|
157
|
+
stream.flush()
|
|
158
|
+
|
|
159
|
+
def animate():
|
|
160
|
+
run_animation(show_prompt=True, stream=sys.stdout)
|
|
161
|
+
|
|
162
|
+
if __name__ == "__main__":
|
|
163
|
+
animate()
|