basira-vision 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- basira_vision/__init__.py +3 -0
- basira_vision/__main__.py +4 -0
- basira_vision/cli.py +623 -0
- basira_vision/pipeline.py +157 -0
- basira_vision/processing/__init__.py +0 -0
- basira_vision/processing/pdf_converter.py +91 -0
- basira_vision/processing/prompt_loader.py +59 -0
- basira_vision/processing/translation.py +59 -0
- basira_vision/processing/typst_processor.py +186 -0
- basira_vision/prompts/ocr/arabic.txt +18 -0
- basira_vision/prompts/ocr/chechen.txt +27 -0
- basira_vision/prompts/ocr/english.txt +14 -0
- basira_vision/prompts/ocr/french.txt +16 -0
- basira_vision/prompts/ocr/german.txt +18 -0
- basira_vision/prompts/ocr/russian.txt +16 -0
- basira_vision/prompts/translate/arabic.txt +41 -0
- basira_vision/prompts/translate/chechen.txt +48 -0
- basira_vision/prompts/translate/english.txt +41 -0
- basira_vision/prompts/translate/french.txt +35 -0
- basira_vision/prompts/translate/german.txt +35 -0
- basira_vision/prompts/translate/russian.txt +42 -0
- basira_vision/providers/__init__.py +14 -0
- basira_vision/providers/base.py +46 -0
- basira_vision/providers/gemini.py +231 -0
- basira_vision/settings.py +37 -0
- basira_vision/templates/template.typ +222 -0
- basira_vision/ui/__init__.py +90 -0
- basira_vision/ui/events.py +116 -0
- basira_vision/ui/live.py +316 -0
- basira_vision/ui/plain.py +124 -0
- basira_vision/ui/state.py +94 -0
- basira_vision/utils/__init__.py +0 -0
- basira_vision/utils/constants.py +24 -0
- basira_vision/utils/errors.py +50 -0
- basira_vision/utils/logging.py +64 -0
- basira_vision/utils/pages.py +210 -0
- basira_vision-0.1.0.dist-info/METADATA +344 -0
- basira_vision-0.1.0.dist-info/RECORD +41 -0
- basira_vision-0.1.0.dist-info/WHEEL +4 -0
- basira_vision-0.1.0.dist-info/entry_points.txt +2 -0
- basira_vision-0.1.0.dist-info/licenses/LICENSE +21 -0
basira_vision/cli.py
ADDED
|
@@ -0,0 +1,623 @@
|
|
|
1
|
+
"""Command-line interface: ``basira ocr | translate | typst``."""
|
|
2
|
+
|
|
3
|
+
import functools
|
|
4
|
+
import shlex
|
|
5
|
+
import shutil
|
|
6
|
+
import sys
|
|
7
|
+
import tempfile
|
|
8
|
+
import time
|
|
9
|
+
from datetime import datetime
|
|
10
|
+
from pathlib import Path
|
|
11
|
+
from typing import Iterable, List, Optional
|
|
12
|
+
|
|
13
|
+
import click
|
|
14
|
+
import pymupdf
|
|
15
|
+
|
|
16
|
+
from . import __version__
|
|
17
|
+
from .pipeline import PageTask, RunResult, run_pages
|
|
18
|
+
from .processing import typst_processor
|
|
19
|
+
from .processing.pdf_converter import IMAGE_FORMATS, PDFRenderer, page_count, parse_page_range
|
|
20
|
+
from .processing.prompt_loader import BUILTIN_PROMPTS_DIR, PromptLoader, read_custom_prompt
|
|
21
|
+
from .processing.translation import (
|
|
22
|
+
check_translation_template,
|
|
23
|
+
default_output_path,
|
|
24
|
+
previous_page_context,
|
|
25
|
+
render_prompt,
|
|
26
|
+
)
|
|
27
|
+
from .providers import VisionProvider, create_provider
|
|
28
|
+
from .settings import Settings
|
|
29
|
+
from .ui import LOG_FORMATS, Hint, Reporter, display_path, print_result_path
|
|
30
|
+
from .ui.events import RunFinished, RunStarted, StepDone, StepStarted
|
|
31
|
+
from .ui.state import pages_label
|
|
32
|
+
from .utils.constants import DEFAULT_TARGET_LANGUAGE, SeparatorType
|
|
33
|
+
from .utils.errors import AuthenticationError, BasiraError, ConfigurationError, EndpointUnreachableError
|
|
34
|
+
from .utils.pages import PageFile, describe_existing, read_page_document
|
|
35
|
+
|
|
36
|
+
PROVIDER_NAME = "Gemini"
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
class CommandError(BasiraError):
|
|
40
|
+
"""An error shown to the user together with what to do about it."""
|
|
41
|
+
|
|
42
|
+
def __init__(self, message: str, hints: Iterable[Hint] = ()):
|
|
43
|
+
super().__init__(message)
|
|
44
|
+
self.hints = list(hints)
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
class Placeholder(str):
|
|
48
|
+
"""A part of a suggested command the user fills in, e.g. ``<language>``; never quoted."""
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def command_line(*args) -> str:
|
|
52
|
+
"""A copy-pasteable ``basira ...`` command; paths are shortened, None arguments dropped."""
|
|
53
|
+
parts = ["basira"]
|
|
54
|
+
for arg in args:
|
|
55
|
+
if arg is None:
|
|
56
|
+
continue
|
|
57
|
+
if isinstance(arg, Placeholder):
|
|
58
|
+
parts.append(arg)
|
|
59
|
+
else:
|
|
60
|
+
parts.append(shlex.quote(display_path(arg) if isinstance(arg, Path) else str(arg)))
|
|
61
|
+
return " ".join(parts)
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def _option(flag: str, value, default=None) -> list:
|
|
65
|
+
return [flag, value] if value is not None and value != default else []
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def _command(stage: str):
|
|
69
|
+
"""Shared options, the reporter's lifetime, error display and the exit code."""
|
|
70
|
+
|
|
71
|
+
def decorate(func):
|
|
72
|
+
@click.option("--log-format", type=click.Choice(LOG_FORMATS), default="auto", show_default=True,
|
|
73
|
+
help="auto: live view in a terminal, plain lines otherwise; json: one event per line")
|
|
74
|
+
@click.option("--verbose", "-v", is_flag=True, help="Show debug details")
|
|
75
|
+
@functools.wraps(func)
|
|
76
|
+
def wrapper(log_format, verbose, **kwargs):
|
|
77
|
+
with Reporter(stage, log_format, verbose) as reporter:
|
|
78
|
+
try:
|
|
79
|
+
code = func(reporter, **kwargs)
|
|
80
|
+
except CommandError as e:
|
|
81
|
+
reporter.error(str(e), e.hints)
|
|
82
|
+
code = 1
|
|
83
|
+
except BasiraError as e:
|
|
84
|
+
reporter.error(str(e))
|
|
85
|
+
code = 1
|
|
86
|
+
except KeyboardInterrupt:
|
|
87
|
+
reporter.error("interrupted before any page was processed")
|
|
88
|
+
code = 130
|
|
89
|
+
sys.exit(code or 0)
|
|
90
|
+
|
|
91
|
+
return wrapper
|
|
92
|
+
|
|
93
|
+
return decorate
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def _check_mode(resume: bool, force_restart: bool) -> None:
|
|
97
|
+
if resume and force_restart:
|
|
98
|
+
raise click.UsageError("--resume and --force-restart cannot be used together")
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def _refuse_overwrite(path: Path, total: Optional[int], resume: bool, force_restart: bool, rerun: List) -> None:
|
|
102
|
+
"""Never replace finished work silently: ask for --resume or --force-restart."""
|
|
103
|
+
if not path.exists() or resume or force_restart:
|
|
104
|
+
return
|
|
105
|
+
done, failed, total = describe_existing(path, total)
|
|
106
|
+
state = f"{done} of {total} pages done" + (f", {len(failed)} failed" if failed else "")
|
|
107
|
+
keep = "keeps it and your edits"
|
|
108
|
+
if failed:
|
|
109
|
+
keep += f", retries {pages_label(failed)}"
|
|
110
|
+
raise CommandError(
|
|
111
|
+
f"{display_path(path)} already exists ({state}). Nothing was changed.",
|
|
112
|
+
[
|
|
113
|
+
Hint("continue", command_line(*rerun, "--resume"), command=True, note=keep),
|
|
114
|
+
Hint("restart", command_line(*rerun, "--force-restart"), command=True, note="deletes it and starts over"),
|
|
115
|
+
],
|
|
116
|
+
)
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
def _connect(settings: Settings, model: Optional[str], reporter: Reporter) -> VisionProvider:
|
|
120
|
+
if not settings.gemini_api_key:
|
|
121
|
+
raise CommandError("GEMINI_API_KEY is not set", [
|
|
122
|
+
Hint("set it", "export GEMINI_API_KEY=your_key", note="or put it in a .env file in this directory"),
|
|
123
|
+
Hint("get one", "https://aistudio.google.com/"),
|
|
124
|
+
])
|
|
125
|
+
try:
|
|
126
|
+
provider = create_provider(
|
|
127
|
+
api_key=settings.gemini_api_key,
|
|
128
|
+
model_name=model or settings.gemini_model,
|
|
129
|
+
base_url=settings.gemini_base_url or None,
|
|
130
|
+
)
|
|
131
|
+
except ValueError as e:
|
|
132
|
+
raise ConfigurationError(str(e)) from e
|
|
133
|
+
|
|
134
|
+
try:
|
|
135
|
+
warning = provider.check_connection()
|
|
136
|
+
except EndpointUnreachableError as e:
|
|
137
|
+
if settings.gemini_base_url:
|
|
138
|
+
hint = Hint("check", f"is the gateway running? curl {settings.gemini_base_url.rstrip('/')}/v1beta/models")
|
|
139
|
+
else:
|
|
140
|
+
hint = Hint("check", "your network connection")
|
|
141
|
+
raise CommandError(f"cannot reach the model endpoint: {e}", [hint]) from None
|
|
142
|
+
except AuthenticationError as e:
|
|
143
|
+
raise CommandError(str(e), [Hint("check", "GEMINI_API_KEY (environment or .env)")]) from None
|
|
144
|
+
if warning:
|
|
145
|
+
reporter.warn(warning)
|
|
146
|
+
return provider
|
|
147
|
+
|
|
148
|
+
|
|
149
|
+
def _model_details(settings: Settings, model: Optional[str], provider: VisionProvider) -> str:
|
|
150
|
+
details = model or settings.gemini_model
|
|
151
|
+
if settings.gemini_base_url:
|
|
152
|
+
details += f" via {provider.endpoint}"
|
|
153
|
+
return details
|
|
154
|
+
|
|
155
|
+
|
|
156
|
+
def _now() -> str:
|
|
157
|
+
return datetime.now().strftime("%Y-%m-%d %H:%M:%S")
|
|
158
|
+
|
|
159
|
+
|
|
160
|
+
def _finish(reporter: Reporter, stage: str, result: RunResult, page_file: PageFile, total: int,
|
|
161
|
+
hints: List[Hint]) -> int:
|
|
162
|
+
reporter.emit(RunFinished(
|
|
163
|
+
stage=stage,
|
|
164
|
+
outcome=result.outcome,
|
|
165
|
+
reason=result.reason,
|
|
166
|
+
done=len(page_file.completed()),
|
|
167
|
+
total=total,
|
|
168
|
+
failed=result.failed,
|
|
169
|
+
elapsed=result.elapsed,
|
|
170
|
+
retries=result.retries,
|
|
171
|
+
hints=hints,
|
|
172
|
+
))
|
|
173
|
+
if result.outcome == "done":
|
|
174
|
+
print_result_path(page_file.path)
|
|
175
|
+
return result.exit_code()
|
|
176
|
+
|
|
177
|
+
|
|
178
|
+
def _resume_hints(result: RunResult, rerun: List) -> List[Hint]:
|
|
179
|
+
resume = command_line(*rerun, "--resume")
|
|
180
|
+
if result.outcome == "paused":
|
|
181
|
+
return [Hint("continue", resume, command=True)]
|
|
182
|
+
if result.outcome == "stopped":
|
|
183
|
+
return [Hint("continue", resume, command=True, note="once the cause is fixed")]
|
|
184
|
+
if result.failed:
|
|
185
|
+
return [Hint("retry", resume, command=True, note="retries only the failed pages")]
|
|
186
|
+
return []
|
|
187
|
+
|
|
188
|
+
|
|
189
|
+
@click.group(context_settings={"help_option_names": ["-h", "--help"]})
|
|
190
|
+
@click.version_option(__version__, prog_name="basira")
|
|
191
|
+
def main():
|
|
192
|
+
"""
|
|
193
|
+
Basira Vision: scanned book in, a new edition in your language out.
|
|
194
|
+
|
|
195
|
+
\b
|
|
196
|
+
1. basira ocr book.pdf --source-language english
|
|
197
|
+
2. review workspace/book/book.md, then:
|
|
198
|
+
basira translate workspace/book/book.md --target-language russian
|
|
199
|
+
3. review workspace/book/book_russian.md, then:
|
|
200
|
+
basira typst workspace/book/book_russian.md --title "..." --author "..."
|
|
201
|
+
|
|
202
|
+
Configuration comes from the environment or a .env file:
|
|
203
|
+
GEMINI_API_KEY (required), GEMINI_MODEL, GEMINI_BASE_URL,
|
|
204
|
+
BASIRA_OUTPUT_DIR, BASIRA_PROMPTS_DIR.
|
|
205
|
+
|
|
206
|
+
Exit codes: 0 done, 3 done with failed pages, 1 error, 2 bad usage,
|
|
207
|
+
130 paused with Ctrl-C.
|
|
208
|
+
"""
|
|
209
|
+
|
|
210
|
+
|
|
211
|
+
# ---------------------------------------------------------------------------- ocr
|
|
212
|
+
|
|
213
|
+
|
|
214
|
+
@main.command()
|
|
215
|
+
@click.argument("pdf_file", type=click.Path(exists=True, dir_okay=False, path_type=Path))
|
|
216
|
+
@click.option("--source-language", default="english", show_default=True,
|
|
217
|
+
help="OCR prompt template, named after the language of the scan")
|
|
218
|
+
@click.option("--page-range", help="Pages to process, e.g. 5 or 1-10 (default: all)")
|
|
219
|
+
@click.option("--dpi", type=click.IntRange(72, 600), default=300, show_default=True,
|
|
220
|
+
help="Render resolution; use 600 for poor scans")
|
|
221
|
+
@click.option("--model", "-m", help="Model name (overrides GEMINI_MODEL)")
|
|
222
|
+
@click.option("--custom-prompt", help="OCR prompt text, or a path to a prompt file")
|
|
223
|
+
@click.option("--prompts-dir", type=click.Path(file_okay=False, path_type=Path),
|
|
224
|
+
help="Extra prompt directory searched before the built-in one (env: BASIRA_PROMPTS_DIR)")
|
|
225
|
+
@click.option("--output-dir", type=click.Path(file_okay=False, path_type=Path),
|
|
226
|
+
help="Workspace directory (default: ./workspace, env: BASIRA_OUTPUT_DIR)")
|
|
227
|
+
@click.option("--image-format", type=click.Choice(IMAGE_FORMATS, case_sensitive=False), default="PNG",
|
|
228
|
+
show_default=True, help="Format of the rendered page images")
|
|
229
|
+
@click.option("--no-images", is_flag=True, help="Do not keep the rendered page images")
|
|
230
|
+
@click.option("--resume", is_flag=True, help="Keep finished pages and process only missing or failed ones")
|
|
231
|
+
@click.option("--force-restart", is_flag=True, help="Delete the existing output and start over")
|
|
232
|
+
@_command("ocr")
|
|
233
|
+
def ocr(reporter, pdf_file, source_language, page_range, dpi, model, custom_prompt, prompts_dir,
|
|
234
|
+
output_dir, image_format, no_images, resume, force_restart):
|
|
235
|
+
"""Read every page of a scanned PDF with a vision model into markdown.
|
|
236
|
+
|
|
237
|
+
Writes <workspace>/<name>/<name>.md with one <!-- Page N --> block per page.
|
|
238
|
+
"""
|
|
239
|
+
_check_mode(resume, force_restart)
|
|
240
|
+
settings = Settings()
|
|
241
|
+
source_language = source_language.strip().lower()
|
|
242
|
+
|
|
243
|
+
total = page_count(pdf_file)
|
|
244
|
+
try:
|
|
245
|
+
requested = parse_page_range(page_range, total)
|
|
246
|
+
except ValueError as e:
|
|
247
|
+
raise click.BadParameter(str(e), param_hint="--page-range") from None
|
|
248
|
+
|
|
249
|
+
if custom_prompt:
|
|
250
|
+
prompt, prompt_label = read_custom_prompt(custom_prompt), "custom"
|
|
251
|
+
else:
|
|
252
|
+
prompt = PromptLoader(prompts_dir or settings.prompts_dir).load("ocr", source_language)
|
|
253
|
+
prompt_label = f"ocr/{source_language}"
|
|
254
|
+
|
|
255
|
+
document_dir = settings.workspace_dir(output_dir) / pdf_file.stem
|
|
256
|
+
output_path = document_dir / f"{pdf_file.stem}.md"
|
|
257
|
+
rerun = [
|
|
258
|
+
"ocr", pdf_file,
|
|
259
|
+
*_option("--page-range", page_range),
|
|
260
|
+
*_option("--source-language", source_language, "english"),
|
|
261
|
+
*_option("--custom-prompt", custom_prompt),
|
|
262
|
+
*_option("--prompts-dir", prompts_dir),
|
|
263
|
+
*_option("--output-dir", output_dir),
|
|
264
|
+
*_option("--model", model),
|
|
265
|
+
*_option("--dpi", dpi, 300),
|
|
266
|
+
*_option("--image-format", image_format.upper(), "PNG"),
|
|
267
|
+
*(["--no-images"] if no_images else []),
|
|
268
|
+
]
|
|
269
|
+
_refuse_overwrite(output_path, total, resume, force_restart, rerun)
|
|
270
|
+
|
|
271
|
+
model_name = model or settings.gemini_model
|
|
272
|
+
provider = _connect(settings, model, reporter)
|
|
273
|
+
replacing = force_restart and output_path.exists()
|
|
274
|
+
page_file = PageFile(
|
|
275
|
+
output_path,
|
|
276
|
+
SeparatorType.OCR,
|
|
277
|
+
{
|
|
278
|
+
"title": "",
|
|
279
|
+
"author": "",
|
|
280
|
+
"source_language": source_language,
|
|
281
|
+
"ai_provider": PROVIDER_NAME,
|
|
282
|
+
"ai_model": model_name,
|
|
283
|
+
"generated": _now(),
|
|
284
|
+
"processing_status": "In progress",
|
|
285
|
+
},
|
|
286
|
+
resume=resume,
|
|
287
|
+
)
|
|
288
|
+
completed = page_file.completed()
|
|
289
|
+
todo = [n for n in requested if n not in completed]
|
|
290
|
+
|
|
291
|
+
details = [f"{total} pages"]
|
|
292
|
+
if page_range:
|
|
293
|
+
details.append(f"range {page_range}")
|
|
294
|
+
if completed:
|
|
295
|
+
details.append(f"{len(completed)} done earlier")
|
|
296
|
+
details += [f"prompt {prompt_label}", _model_details(settings, model, provider), f"{dpi} dpi"]
|
|
297
|
+
reporter.emit(RunStarted("ocr", display_path(pdf_file), display_path(output_path), " · ".join(details),
|
|
298
|
+
pages=list(range(1, total + 1)), todo=todo, done_before=sorted(completed)))
|
|
299
|
+
if replacing:
|
|
300
|
+
reporter.warn("starting over: the previous output was deleted")
|
|
301
|
+
if not todo:
|
|
302
|
+
reporter.warn("nothing to do: every requested page is already done")
|
|
303
|
+
|
|
304
|
+
images_dir = None if no_images else document_dir / "images"
|
|
305
|
+
with tempfile.TemporaryDirectory(prefix="basira_") as tmp, PDFRenderer(pdf_file, dpi, image_format) as renderer:
|
|
306
|
+
target_dir = images_dir or Path(tmp)
|
|
307
|
+
|
|
308
|
+
def read_page(number: int):
|
|
309
|
+
return lambda: provider.extract_text(renderer.render(number, target_dir), prompt)
|
|
310
|
+
|
|
311
|
+
tasks = [PageTask(n, read_page(n), "reading") for n in todo]
|
|
312
|
+
result = run_pages(tasks, page_file, provider, reporter, set(range(1, total + 1)))
|
|
313
|
+
|
|
314
|
+
hints = _resume_hints(result, rerun)
|
|
315
|
+
outside = set(range(1, total + 1)) - page_file.completed() - set(page_file.failed()) - set(requested)
|
|
316
|
+
if outside and result.outcome == "done":
|
|
317
|
+
first, last = min(outside), max(outside)
|
|
318
|
+
if last - first + 1 == len(outside):
|
|
319
|
+
without_range = [arg for i, arg in enumerate(rerun)
|
|
320
|
+
if arg != "--page-range" and (i == 0 or rerun[i - 1] != "--page-range")]
|
|
321
|
+
span = str(first) if first == last else f"{first}-{last}"
|
|
322
|
+
hints.append(Hint("remaining", command_line(*without_range, "--page-range", span, "--resume"),
|
|
323
|
+
command=True, note=f"{pages_label(outside)} not processed yet"))
|
|
324
|
+
else:
|
|
325
|
+
hints.append(Hint("remaining", f"{pages_label(outside)} not processed yet",
|
|
326
|
+
note="process them with --page-range … --resume"))
|
|
327
|
+
hints.append(Hint("output", display_path(output_path)))
|
|
328
|
+
if images_dir:
|
|
329
|
+
hints.append(Hint("images", display_path(images_dir)))
|
|
330
|
+
if result.outcome == "done" and not page_file.failed() and not outside:
|
|
331
|
+
next_step = command_line("translate", output_path, "--target-language", Placeholder("<language>"))
|
|
332
|
+
hints.append(Hint("next", next_step, command=True, note="after reviewing the markdown"))
|
|
333
|
+
return _finish(reporter, "ocr", result, page_file, total, hints)
|
|
334
|
+
|
|
335
|
+
|
|
336
|
+
# ---------------------------------------------------------------------------- translate
|
|
337
|
+
|
|
338
|
+
|
|
339
|
+
@main.command()
|
|
340
|
+
@click.argument("markdown_file", type=click.Path(exists=True, dir_okay=False, path_type=Path))
|
|
341
|
+
@click.option("--target-language", default=DEFAULT_TARGET_LANGUAGE, show_default=True,
|
|
342
|
+
help="Translation prompt template, named after the target language")
|
|
343
|
+
@click.option("--model", "-m", help="Model name (overrides GEMINI_MODEL)")
|
|
344
|
+
@click.option("--custom-prompt",
|
|
345
|
+
help="Translation prompt text or file; must contain {text}, "
|
|
346
|
+
"may use {context_section} and {source_language}")
|
|
347
|
+
@click.option("--prompts-dir", type=click.Path(file_okay=False, path_type=Path),
|
|
348
|
+
help="Extra prompt directory searched before the built-in one (env: BASIRA_PROMPTS_DIR)")
|
|
349
|
+
@click.option("--output", "-o", type=click.Path(dir_okay=False, path_type=Path),
|
|
350
|
+
help="Output markdown (default: <name>_<language>.md next to the input)")
|
|
351
|
+
@click.option("--resume", is_flag=True, help="Keep finished pages and translate only missing or failed ones")
|
|
352
|
+
@click.option("--force-restart", is_flag=True, help="Delete the existing translation and start over")
|
|
353
|
+
@_command("translate")
|
|
354
|
+
def translate(reporter, markdown_file, target_language, model, custom_prompt, prompts_dir, output,
|
|
355
|
+
resume, force_restart):
|
|
356
|
+
"""Translate an OCR markdown file page by page with a sliding window.
|
|
357
|
+
|
|
358
|
+
Each page is sent with the previous page's finished translation as
|
|
359
|
+
context. A failed page clears the window for the next one.
|
|
360
|
+
"""
|
|
361
|
+
_check_mode(resume, force_restart)
|
|
362
|
+
settings = Settings()
|
|
363
|
+
target_language = target_language.strip().lower()
|
|
364
|
+
loader = PromptLoader(prompts_dir or settings.prompts_dir)
|
|
365
|
+
|
|
366
|
+
if custom_prompt:
|
|
367
|
+
template = read_custom_prompt(custom_prompt)
|
|
368
|
+
else:
|
|
369
|
+
template = loader.load("translate", target_language)
|
|
370
|
+
check_translation_template(template)
|
|
371
|
+
|
|
372
|
+
source_meta, pages = read_page_document(markdown_file.read_text(encoding="utf-8"))
|
|
373
|
+
if not pages:
|
|
374
|
+
raise CommandError(f"no pages found in {display_path(markdown_file)}")
|
|
375
|
+
source_language = str(source_meta.get("source_language") or "").strip()
|
|
376
|
+
|
|
377
|
+
output_path = output or default_output_path(markdown_file, target_language, loader.available("translate"))
|
|
378
|
+
rerun = [
|
|
379
|
+
"translate", markdown_file,
|
|
380
|
+
"--target-language", target_language,
|
|
381
|
+
*_option("--custom-prompt", custom_prompt),
|
|
382
|
+
*_option("--prompts-dir", prompts_dir),
|
|
383
|
+
*_option("--model", model),
|
|
384
|
+
*_option("--output", output),
|
|
385
|
+
]
|
|
386
|
+
_refuse_overwrite(output_path, len(pages), resume, force_restart, rerun)
|
|
387
|
+
|
|
388
|
+
model_name = model or settings.gemini_model
|
|
389
|
+
provider = _connect(settings, model, reporter)
|
|
390
|
+
replacing = force_restart and output_path.exists()
|
|
391
|
+
try:
|
|
392
|
+
source_file = str(markdown_file.resolve().relative_to(output_path.resolve().parent))
|
|
393
|
+
except ValueError:
|
|
394
|
+
source_file = str(markdown_file.resolve())
|
|
395
|
+
page_file = PageFile(
|
|
396
|
+
output_path,
|
|
397
|
+
SeparatorType.TYPST,
|
|
398
|
+
{
|
|
399
|
+
"title": source_meta.get("title") or "",
|
|
400
|
+
"author": source_meta.get("author") or "",
|
|
401
|
+
"source_file": source_file,
|
|
402
|
+
"source_language": source_language or None,
|
|
403
|
+
"target_language": target_language,
|
|
404
|
+
"ai_provider": PROVIDER_NAME,
|
|
405
|
+
"ai_model": model_name,
|
|
406
|
+
"translated": _now(),
|
|
407
|
+
"processing_status": "In progress",
|
|
408
|
+
},
|
|
409
|
+
resume=resume,
|
|
410
|
+
)
|
|
411
|
+
completed = page_file.completed()
|
|
412
|
+
todo = [i for i, page in enumerate(pages) if page.number not in completed]
|
|
413
|
+
|
|
414
|
+
details = [f"{len(pages)} pages"]
|
|
415
|
+
if completed:
|
|
416
|
+
details.append(f"{len(completed & {p.number for p in pages})} done earlier")
|
|
417
|
+
details += [f"{source_language or 'auto'} → {target_language}", _model_details(settings, model, provider)]
|
|
418
|
+
reporter.emit(RunStarted("translate", display_path(markdown_file), display_path(output_path),
|
|
419
|
+
" · ".join(details), pages=[p.number for p in pages],
|
|
420
|
+
todo=[pages[i].number for i in todo], done_before=sorted(completed)))
|
|
421
|
+
if replacing:
|
|
422
|
+
reporter.warn("starting over: the previous translation was deleted")
|
|
423
|
+
if not todo:
|
|
424
|
+
reporter.warn("nothing to do: every page is already translated")
|
|
425
|
+
|
|
426
|
+
source_display = source_language.capitalize() if source_language else "Auto"
|
|
427
|
+
|
|
428
|
+
def context_note(index: int) -> str:
|
|
429
|
+
if index == 0:
|
|
430
|
+
return "translating · first page, no context"
|
|
431
|
+
previous = pages[index - 1]
|
|
432
|
+
if previous_page_context(pages, index, page_file.pages) is not None:
|
|
433
|
+
return f"translating · context: page {previous.number}"
|
|
434
|
+
return f"translating · context cleared (page {previous.number} has no translation)"
|
|
435
|
+
|
|
436
|
+
def translate_page(index: int):
|
|
437
|
+
page = pages[index]
|
|
438
|
+
|
|
439
|
+
def work() -> str:
|
|
440
|
+
context = previous_page_context(pages, index, page_file.pages)
|
|
441
|
+
return provider.generate_text(render_prompt(template, page.text, context, source_display))
|
|
442
|
+
|
|
443
|
+
return work
|
|
444
|
+
|
|
445
|
+
tasks = []
|
|
446
|
+
for index in todo:
|
|
447
|
+
page = pages[index]
|
|
448
|
+
if page.is_error:
|
|
449
|
+
tasks.append(PageTask(page.number, skip_reason=f"no OCR text in the source ({page.error})"))
|
|
450
|
+
elif not page.text:
|
|
451
|
+
tasks.append(PageTask(page.number, lambda: "", "empty page"))
|
|
452
|
+
else:
|
|
453
|
+
tasks.append(PageTask(page.number, translate_page(index), functools.partial(context_note, index)))
|
|
454
|
+
result = run_pages(tasks, page_file, provider, reporter, {p.number for p in pages})
|
|
455
|
+
|
|
456
|
+
hints = _resume_hints(result, rerun)
|
|
457
|
+
if any(page.is_error for page in pages):
|
|
458
|
+
hints.append(Hint("source", f"fix the failed OCR pages in {display_path(markdown_file)} first",
|
|
459
|
+
note="write the text under their markers, then rerun with --resume"))
|
|
460
|
+
hints.append(Hint("output", display_path(output_path)))
|
|
461
|
+
if result.outcome == "done" and not page_file.failed():
|
|
462
|
+
hints.append(Hint("next", command_line("typst", display_path(output_path)), command=True,
|
|
463
|
+
note="after reviewing the translation"))
|
|
464
|
+
return _finish(reporter, "translate", result, page_file, len(pages), hints)
|
|
465
|
+
|
|
466
|
+
|
|
467
|
+
# ---------------------------------------------------------------------------- typst
|
|
468
|
+
|
|
469
|
+
|
|
470
|
+
@main.command()
|
|
471
|
+
@click.argument("markdown_file", type=click.Path(exists=True, dir_okay=False, path_type=Path))
|
|
472
|
+
@click.option("--title", help="Book title (overrides the frontmatter)")
|
|
473
|
+
@click.option("--author", help="Book author (overrides the frontmatter)")
|
|
474
|
+
@click.option("--subtitle", help="Subtitle under the title")
|
|
475
|
+
@click.option("--version", help="Edition line above the date, e.g. 'Second edition'")
|
|
476
|
+
@click.option("--date", help="Date on the title page (default: current year)")
|
|
477
|
+
@click.option("--translator-preface", type=click.Path(exists=True, dir_okay=False, path_type=Path),
|
|
478
|
+
help="Markdown preface inserted after the title page")
|
|
479
|
+
@click.option("--lang", help="Document language code, e.g. en, ru (default: from the frontmatter)")
|
|
480
|
+
@click.option("--header", help="Running header text (default: the title)")
|
|
481
|
+
@click.option("--toc-title", help="Table of contents heading (default depends on --lang)")
|
|
482
|
+
@click.option("--toc/--no-toc", default=None, help="Show or hide the table of contents")
|
|
483
|
+
@click.option("--template", type=click.Path(exists=True, dir_okay=False, path_type=Path),
|
|
484
|
+
help="Your own copy of the template (start from `basira export template`)")
|
|
485
|
+
@click.option("--output", "-o", type=click.Path(dir_okay=False, path_type=Path),
|
|
486
|
+
help="Output PDF (default: next to the markdown)")
|
|
487
|
+
@_command("typst")
|
|
488
|
+
def typst(reporter, markdown_file, title, author, translator_preface, toc, template, output, **variables):
|
|
489
|
+
"""Typeset a markdown file into a book PDF with Pandoc and Typst.
|
|
490
|
+
|
|
491
|
+
Title and author come from the YAML frontmatter unless given here.
|
|
492
|
+
"""
|
|
493
|
+
missing_tools = typst_processor.missing_tools()
|
|
494
|
+
if missing_tools:
|
|
495
|
+
raise CommandError(f"{' and '.join(missing_tools)} not found on PATH", [
|
|
496
|
+
Hint("pandoc", "https://pandoc.org/installing.html"),
|
|
497
|
+
Hint("typst", "https://github.com/typst/typst#installation"),
|
|
498
|
+
])
|
|
499
|
+
|
|
500
|
+
content = markdown_file.read_text(encoding="utf-8")
|
|
501
|
+
meta, pages = read_page_document(content)
|
|
502
|
+
metadata = typst_processor.resolve_metadata(
|
|
503
|
+
content, title, author, str(translator_preface) if translator_preface else None, variables.get("lang")
|
|
504
|
+
)
|
|
505
|
+
missing = [name for name in ("title", "author") if not metadata["title" if name == "title" else "authors"]]
|
|
506
|
+
if missing:
|
|
507
|
+
raise CommandError(f"no {' and '.join(missing)} for {display_path(markdown_file)}", [
|
|
508
|
+
Hint("add", f"{' and '.join(f'{m}:' for m in missing)} to its YAML frontmatter"),
|
|
509
|
+
Hint("or pass", command_line("typst", markdown_file, "--title", Placeholder('"..."'),
|
|
510
|
+
"--author", Placeholder('"..."')), command=True),
|
|
511
|
+
])
|
|
512
|
+
|
|
513
|
+
output_path = output or markdown_file.with_suffix(".pdf")
|
|
514
|
+
template_name = display_path(template) if template else "A5 template"
|
|
515
|
+
details = [metadata["title"], ", ".join(metadata["authors"]), metadata["lang"] or "en", template_name]
|
|
516
|
+
reporter.emit(RunStarted("typst", display_path(markdown_file), display_path(output_path), " · ".join(details)))
|
|
517
|
+
|
|
518
|
+
failed = sorted(p.number for p in pages if p.is_error)
|
|
519
|
+
if failed:
|
|
520
|
+
retry = ""
|
|
521
|
+
if meta.get("source_file") and meta.get("target_language"):
|
|
522
|
+
source = display_path(markdown_file.parent / str(meta["source_file"]))
|
|
523
|
+
retry = "retry first: " + command_line("translate", source, "--target-language",
|
|
524
|
+
meta["target_language"], "--resume")
|
|
525
|
+
verb = "is" if len(failed) == 1 else "are"
|
|
526
|
+
reporter.warn(f"{pages_label(failed)} failed in an earlier step and {verb} left out of the book", hint=retry)
|
|
527
|
+
elif str(meta.get("processing_status", "")).lower() == "incomplete":
|
|
528
|
+
reporter.warn("the file is marked Incomplete, so some pages may be missing from the book")
|
|
529
|
+
|
|
530
|
+
started = time.monotonic()
|
|
531
|
+
|
|
532
|
+
def on_step(name: str, seconds: Optional[float]) -> None:
|
|
533
|
+
reporter.emit(StepStarted(name) if seconds is None else StepDone(name, seconds))
|
|
534
|
+
|
|
535
|
+
typst_processor.generate_pdf(
|
|
536
|
+
markdown_file,
|
|
537
|
+
output_path,
|
|
538
|
+
title=title,
|
|
539
|
+
author=author,
|
|
540
|
+
translator_preface=str(translator_preface) if translator_preface else None,
|
|
541
|
+
toc=toc,
|
|
542
|
+
template_path=template or typst_processor.TEMPLATE_PATH,
|
|
543
|
+
on_step=on_step,
|
|
544
|
+
**variables,
|
|
545
|
+
)
|
|
546
|
+
with pymupdf.open(output_path) as pdf:
|
|
547
|
+
pdf_pages = len(pdf)
|
|
548
|
+
size_kb = output_path.stat().st_size / 1024
|
|
549
|
+
reporter.emit(RunFinished(
|
|
550
|
+
stage="typst",
|
|
551
|
+
outcome="done",
|
|
552
|
+
elapsed=time.monotonic() - started,
|
|
553
|
+
extra=f"{pdf_pages} pages · {size_kb:,.0f} KB",
|
|
554
|
+
hints=[Hint("output", display_path(output_path))],
|
|
555
|
+
))
|
|
556
|
+
print_result_path(output_path)
|
|
557
|
+
return 0
|
|
558
|
+
|
|
559
|
+
|
|
560
|
+
# ---------------------------------------------------------------------------- export
|
|
561
|
+
|
|
562
|
+
|
|
563
|
+
@main.group()
|
|
564
|
+
def export():
|
|
565
|
+
"""Write copies of the built-in template or prompts, to edit and use as your own."""
|
|
566
|
+
|
|
567
|
+
|
|
568
|
+
@export.command("template")
|
|
569
|
+
@click.argument("destination", required=False, default="template.typ",
|
|
570
|
+
type=click.Path(dir_okay=False, path_type=Path))
|
|
571
|
+
@click.option("--force", is_flag=True, help="Overwrite the file if it exists")
|
|
572
|
+
@_command("export")
|
|
573
|
+
def export_template(reporter, destination, force):
|
|
574
|
+
"""Copy the book template to DESTINATION (default: ./template.typ).
|
|
575
|
+
|
|
576
|
+
Use it with: basira typst book.md --template template.typ
|
|
577
|
+
"""
|
|
578
|
+
if destination.exists() and not force:
|
|
579
|
+
raise CommandError(f"{display_path(destination)} already exists. Nothing was changed.", [
|
|
580
|
+
Hint("overwrite", command_line("export", "template", destination, "--force"), command=True),
|
|
581
|
+
])
|
|
582
|
+
destination.parent.mkdir(parents=True, exist_ok=True)
|
|
583
|
+
shutil.copyfile(typst_processor.TEMPLATE_PATH, destination)
|
|
584
|
+
reporter.emit(RunFinished(stage="export", outcome="done", extra="template written", hints=[
|
|
585
|
+
Hint("file", display_path(destination)),
|
|
586
|
+
Hint("use it", command_line("typst", Placeholder("book.md"), "--template", destination), command=True),
|
|
587
|
+
]))
|
|
588
|
+
print_result_path(destination)
|
|
589
|
+
return 0
|
|
590
|
+
|
|
591
|
+
|
|
592
|
+
@export.command("prompts")
|
|
593
|
+
@click.argument("destination", required=False, default="prompts",
|
|
594
|
+
type=click.Path(file_okay=False, path_type=Path))
|
|
595
|
+
@click.option("--force", is_flag=True, help="Overwrite prompt files that already exist")
|
|
596
|
+
@_command("export")
|
|
597
|
+
def export_prompts(reporter, destination, force):
|
|
598
|
+
"""Copy all OCR and translation prompts to DESTINATION (default: ./prompts).
|
|
599
|
+
|
|
600
|
+
Use them with --prompts-dir DESTINATION (or BASIRA_PROMPTS_DIR).
|
|
601
|
+
"""
|
|
602
|
+
sources = sorted(BUILTIN_PROMPTS_DIR.rglob("*.txt"))
|
|
603
|
+
targets = [(source, destination / source.relative_to(BUILTIN_PROMPTS_DIR)) for source in sources]
|
|
604
|
+
existing = [target for _, target in targets if target.exists()]
|
|
605
|
+
if existing and not force:
|
|
606
|
+
raise CommandError(
|
|
607
|
+
f"{len(existing)} prompt file(s) already exist in {display_path(destination)}. Nothing was changed.",
|
|
608
|
+
[Hint("overwrite", command_line("export", "prompts", destination, "--force"), command=True)],
|
|
609
|
+
)
|
|
610
|
+
for source, target in targets:
|
|
611
|
+
target.parent.mkdir(parents=True, exist_ok=True)
|
|
612
|
+
shutil.copyfile(source, target)
|
|
613
|
+
reporter.emit(RunFinished(stage="export", outcome="done", extra=f"{len(targets)} prompts written", hints=[
|
|
614
|
+
Hint("folder", display_path(destination)),
|
|
615
|
+
Hint("use them", command_line("ocr", Placeholder("book.pdf"), "--prompts-dir", destination), command=True,
|
|
616
|
+
note="same for translate; or set BASIRA_PROMPTS_DIR"),
|
|
617
|
+
]))
|
|
618
|
+
print_result_path(destination)
|
|
619
|
+
return 0
|
|
620
|
+
|
|
621
|
+
|
|
622
|
+
if __name__ == "__main__":
|
|
623
|
+
main()
|