synthocr 2.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
synthocr/web.py ADDED
@@ -0,0 +1,2259 @@
1
+ """A local FastAPI studio over the maintained, bounded SynthOCR pipeline."""
2
+
3
+ import base64
4
+ import hashlib
5
+ import heapq
6
+ import io
7
+ import json
8
+ import ntpath
9
+ import os
10
+ import re
11
+ import shutil
12
+ import sqlite3
13
+ import struct
14
+ import tempfile
15
+ import threading
16
+ import time
17
+ import zlib
18
+ from collections import OrderedDict
19
+ from concurrent.futures import ThreadPoolExecutor
20
+ from contextlib import asynccontextmanager
21
+ from dataclasses import replace
22
+ from datetime import UTC, datetime
23
+ from importlib.resources import files
24
+ from math import isfinite
25
+ from pathlib import Path
26
+ from typing import Annotated, Literal
27
+ from urllib.parse import urlsplit
28
+ from uuid import uuid4
29
+
30
+ from anyio import CancelScope
31
+ from fastapi import FastAPI, HTTPException, Query, Request
32
+ from fastapi import Path as FastAPIPath
33
+ from fastapi.responses import FileResponse, JSONResponse, Response
34
+ from fastapi.staticfiles import StaticFiles
35
+ from fontTools.ttLib import TTCollection, TTFont, TTLibError
36
+ from PIL import Image, ImageFont, features
37
+ from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
38
+ from starlette.concurrency import run_in_threadpool
39
+ from starlette.middleware.trustedhost import TrustedHostMiddleware
40
+
41
+ from . import __version__
42
+ from .config import (
43
+ AUGMENTATION_FIELDS,
44
+ DOCUMENT_LAYER_LABELS,
45
+ DOCUMENT_PRESETS,
46
+ IMAGE_PRESETS,
47
+ Config,
48
+ background_distribution,
49
+ validate_augmentation,
50
+ )
51
+ from .corpus_store import canonical_language, resolve_source_language
52
+ from .downloads import sha256
53
+ from .fonts import FONT_SUFFIXES, Coverage, discover, font_ids, select
54
+ from .pipeline import GenerationCancelled, export, generate
55
+ from .render import Renderer
56
+ from .spelling import spelling_cases
57
+ from .text import chunks, normalize, prepare, records, required_codepoints
58
+ from .verify import dataset_files, verify
59
+
60
+ NAME = re.compile(r"[A-Za-z0-9][A-Za-z0-9_-]{0,63}\Z")
61
+ LOCATION = re.compile(r"[a-z][a-z0-9-]{0,31}\Z")
62
+ FOLDER_NAME = re.compile(r'[^\x00-\x1f\x7f<>:"/\\|?*]{1,80}\Z')
63
+ DIRECTORY_LIMIT = 200
64
+ DIRECTORY_SCAN_LIMIT = 2000
65
+ JOB_ID = re.compile(r"[a-f0-9]{32}\Z")
66
+ DOWNLOAD_FORMAT = Literal["dataset", "csv", "imagefolder", "tesseract", "manifest"]
67
+ LANGUAGE = re.compile(r"[A-Za-z]{2,8}(?:-[A-Za-z0-9]{1,8})*\Z")
68
+ IMAGE = re.compile(r"[0-9]{8}\.(?:png|jpg|webp|tif)\Z")
69
+ CACHE_FILES = 128
70
+ INDEX_STRIDE = 128
71
+ UPLOAD_BYTES = 50 * 1024 * 1024
72
+ FONT_UPLOAD_BYTES = 32 * 1024 * 1024
73
+ USER_FONT_LIMIT = 200
74
+ FONT_ID = re.compile(r"[a-f0-9]{64}:[0-9]{1,4}\Z")
75
+ # A short sample of each page language; a font that draws it all is marked as
76
+ # supporting that language. Generation still checks every label separately.
77
+ LANGUAGE_SAMPLES = {
78
+ "ckb": "ئابپتجچحخدرڕزژسشعغفڤقکگلڵمنوۆهەیێ",
79
+ "kmr": "abcçdeêfghiîjklmnopqrsştuûvwxyzABCÇDEÊFGHIÎJKLMNOPQRSŞTUÛVWXYZ",
80
+ "ar": "ابتثجحخدذرزسشصضطظعغفقكلمنهوي",
81
+ "en": "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ",
82
+ }
83
+ UPLOAD_TOTAL_BYTES = 100 * 1024 * 1024
84
+ UPLOAD_FILES = 8
85
+ UPLOAD_TTL = 24 * 60 * 60
86
+ UPLOAD_CHUNK = 64 * 1024
87
+ ACTIVE = {"queued", "preparing", "generating", "verifying", "cancelling"}
88
+ LANGUAGES = {"sorani": ["ckb"], "kurmanji": ["kmr"], "both": ["ckb", "kmr"], "all": []}
89
+ FOCUSED_LANGUAGES = ("ckb", "kmr", "ar", "en")
90
+ LanguageSelection = Annotated[
91
+ list[Literal["ckb", "kmr", "ar", "en"]], Field(min_length=1, max_length=4)
92
+ ]
93
+
94
+
95
+ def _font_display_name(font) -> str:
96
+ try:
97
+ with TTFont(font.path, fontNumber=font.index, lazy=True) as face:
98
+ name = face["name"].getDebugName(4) or face["name"].getDebugName(1)
99
+ except OSError, ValueError, KeyError, TTLibError:
100
+ name = None
101
+ return (name or Path(font.path).stem)[:80]
102
+
103
+
104
+ def _folder_name(name: str) -> bool:
105
+ if not isinstance(name, str):
106
+ return False
107
+ try:
108
+ if len(name.encode("utf-8")) > 255:
109
+ return False
110
+ except UnicodeEncodeError:
111
+ return False
112
+ return bool(
113
+ FOLDER_NAME.fullmatch(name)
114
+ and not name.startswith(".")
115
+ and name[-1] not in " ."
116
+ and not ntpath.isreserved(name)
117
+ )
118
+
119
+
120
+ def _folder_parts(path: str) -> list[str]:
121
+ if not isinstance(path, str) or len(path) > 512:
122
+ raise ValueError("Output folder must be a relative path of at most 512 characters")
123
+ parts = path.split("/") if path else []
124
+ if len(parts) > 8 or any(not _folder_name(part) for part in parts):
125
+ raise ValueError("Choose an output folder within the selected storage location")
126
+ return parts
127
+
128
+
129
+ class DirectoryRequest(BaseModel):
130
+ model_config = ConfigDict(extra="forbid")
131
+
132
+ location: str = Field(default="default", pattern=r"^[a-z][a-z0-9-]{0,31}$")
133
+ path: str = Field(default="", max_length=512)
134
+ name: str = Field(min_length=1, max_length=80)
135
+
136
+ @model_validator(mode="after")
137
+ def valid_directory(self):
138
+ _folder_parts(self.path)
139
+ if not _folder_name(self.name):
140
+ raise ValueError("Use a portable folder name without path separators or reserved names")
141
+ _folder_parts("/".join(filter(None, (self.path, self.name))))
142
+ return self
143
+
144
+
145
+ class AppearanceRequest(BaseModel):
146
+ model_config = ConfigDict(extra="forbid")
147
+
148
+ augmentation: float = Field(default=0.35, ge=0, le=1, allow_inf_nan=False, strict=True)
149
+ background: Literal["paper", "plain", "gradient", "noise"] = "paper"
150
+ color_mode: Literal["grayscale", "rgb"] = "grayscale"
151
+ background_color: str = "#f5f1e8"
152
+ background_color_end: str = "#e4e9ed"
153
+ text_color: str = "#1a1a1a"
154
+ texture_strength: float = Field(default=0.04, ge=0, le=0.25, allow_inf_nan=False, strict=True)
155
+ rotation: float = Field(default=2, ge=0, le=10, allow_inf_nan=False, strict=True)
156
+ blur: float = Field(default=0.55, ge=0, le=2, allow_inf_nan=False, strict=True)
157
+ contrast: float = Field(default=0.1, ge=0, le=0.5, allow_inf_nan=False, strict=True)
158
+ background_presets: list[str] = Field(default_factory=list, max_length=len(DOCUMENT_PRESETS))
159
+ background_weights: dict[
160
+ str, Annotated[float, Field(ge=0, le=1_000_000, allow_inf_nan=False, strict=True)]
161
+ ] = Field(default_factory=dict, max_length=len(DOCUMENT_PRESETS))
162
+ preset_colors: bool = Field(default=True, strict=True)
163
+ paper_texture: float | None = Field(default=None, ge=0, le=1, allow_inf_nan=False, strict=True)
164
+ grain: float | None = Field(default=None, ge=0, le=1, allow_inf_nan=False, strict=True)
165
+ shadow: float | None = Field(default=None, ge=0, le=1, allow_inf_nan=False, strict=True)
166
+ stains: float | None = Field(default=None, ge=0, le=1, allow_inf_nan=False, strict=True)
167
+ creases: float | None = Field(default=None, ge=0, le=1, allow_inf_nan=False, strict=True)
168
+ ruled_lines: float | None = Field(default=None, ge=0, le=1, allow_inf_nan=False, strict=True)
169
+ scan_banding: float | None = Field(default=None, ge=0, le=1, allow_inf_nan=False, strict=True)
170
+ speckles: float | None = Field(default=None, ge=0, le=1, allow_inf_nan=False, strict=True)
171
+ binding_shadow: float | None = Field(default=None, ge=0, le=1, allow_inf_nan=False, strict=True)
172
+
173
+ @model_validator(mode="after")
174
+ def valid_appearance(self):
175
+ validate_augmentation(self)
176
+ return self
177
+
178
+
179
+ class JobRequest(AppearanceRequest):
180
+ name: str | None = Field(default=None, min_length=1, max_length=64)
181
+ output_location: str = Field(default="default", pattern=r"^[a-z][a-z0-9-]{0,31}$")
182
+ output_folder: str = Field(default="", max_length=512)
183
+ language: str = "sorani"
184
+ languages: LanguageSelection | None = None
185
+ source: Literal["corpus", "custom", "spelling", "upload"] = "corpus"
186
+ language_weights: dict[str, float] = Field(default_factory=dict, max_length=8)
187
+ upload_id: str | None = Field(default=None, min_length=32, max_length=32)
188
+ text: str = Field(default="", max_length=200_000)
189
+ samples: int = Field(default=1000, ge=1, le=100_000, strict=True)
190
+ font_size: int = Field(default=48, ge=12, le=128, strict=True)
191
+ fonts: list[str] = Field(default_factory=list, max_length=1000)
192
+
193
+ @field_validator("fonts")
194
+ @classmethod
195
+ def chosen_font_ids(cls, value):
196
+ if any(not FONT_ID.fullmatch(item) for item in value):
197
+ raise ValueError("Fonts are chosen by their ID from /api/fonts")
198
+ return value
199
+
200
+ format: Literal["png", "jpg", "webp", "tif"] = "png"
201
+ seed: int = Field(default=42, ge=-(2**63), le=2**63 - 1, strict=True)
202
+ workers: int = Field(default=1, ge=1, le=4, strict=True)
203
+ train_ratio: float = Field(default=0.9, ge=0, le=1, allow_inf_nan=False)
204
+ validation_ratio: float = Field(default=0.05, ge=0, le=1, allow_inf_nan=False)
205
+
206
+ @model_validator(mode="after")
207
+ def coherent(self):
208
+ if self.name is not None and (not NAME.fullmatch(self.name) or not _folder_name(self.name)):
209
+ raise ValueError("Dataset name may contain letters, numbers, underscores and hyphens")
210
+ _folder_parts(self.output_folder)
211
+ if self.language not in LANGUAGES and not LANGUAGE.fullmatch(self.language):
212
+ raise ValueError("Choose Sorani, Kurmanji, both, all or a valid source language tag")
213
+ tags = _language_tags(self.language, self.languages)
214
+ if self.language_weights and (
215
+ not set(self.language_weights) <= set(tags)
216
+ or any(
217
+ not isfinite(value) or not 0 <= value <= 1_000_000
218
+ for value in self.language_weights.values()
219
+ )
220
+ or not any(self.language_weights.get(tag, 1) > 0 for tag in tags)
221
+ ):
222
+ raise ValueError(
223
+ "Language shares must be numbers for the selected languages, "
224
+ "with at least one above zero"
225
+ )
226
+ if self.train_ratio + self.validation_ratio > 1:
227
+ raise ValueError("Training and validation proportions must add up to at most 1")
228
+ if len(self.text.encode("utf-8")) > 200 * 1024:
229
+ raise ValueError("Custom text must be at most 200 KiB")
230
+ if self.source == "custom":
231
+ if len(tags) != 1:
232
+ raise ValueError("Choose one explicit source language for your own text")
233
+ if not required_codepoints(self.text):
234
+ raise ValueError("Enter at least one line containing visible text or numbers")
235
+ elif self.text:
236
+ raise ValueError("Custom text is only accepted with the custom text source")
237
+ if self.source == "spelling" and (
238
+ not tags or any(tag not in {"ckb", "kmr"} for tag in tags)
239
+ ):
240
+ raise ValueError("Spelling checks support Sorani, Kurmanji or both")
241
+ if self.source == "upload":
242
+ if self.upload_id is None or not JOB_ID.fullmatch(self.upload_id):
243
+ raise ValueError("Upload a text file before generating")
244
+ elif self.upload_id is not None:
245
+ raise ValueError("Uploaded files are only accepted with the upload source")
246
+ return self
247
+
248
+
249
+ class PreviewRequest(AppearanceRequest):
250
+ model_config = ConfigDict(extra="forbid")
251
+
252
+ text: str = Field(default="", max_length=200_000)
253
+ language: str = "sorani"
254
+ languages: LanguageSelection | None = None
255
+ source: Literal["demo", "corpus", "spelling"] = "demo"
256
+ index: int = Field(default=0, ge=0, le=31, strict=True)
257
+ font_size: int = Field(default=48, ge=12, le=128, strict=True)
258
+ augmentation: float = Field(default=0, ge=0, le=1, allow_inf_nan=False, strict=True)
259
+ seed: int = Field(default=42, ge=-(2**63), le=2**63 - 1, strict=True)
260
+ compare: bool = Field(default=False, strict=True)
261
+ fonts: list[str] = Field(default_factory=list, max_length=1000)
262
+
263
+ @field_validator("fonts")
264
+ @classmethod
265
+ def chosen_font_ids(cls, value):
266
+ if any(not FONT_ID.fullmatch(item) for item in value):
267
+ raise ValueError("Fonts are chosen by their ID from /api/fonts")
268
+ return value
269
+
270
+ @model_validator(mode="after")
271
+ def bounded_text(self):
272
+ if len(self.text.encode("utf-8")) > 200 * 1024:
273
+ raise ValueError("Preview text must be at most 200 KiB")
274
+ tags = _language_tags(self.language, self.languages)
275
+ if self.languages is not None and self.source == "demo" and self.text and len(tags) != 1:
276
+ raise ValueError("Choose one explicit source language for your preview text")
277
+ if self.source == "spelling" and (
278
+ not tags or any(tag not in {"ckb", "kmr"} for tag in tags)
279
+ ):
280
+ raise ValueError("Spelling examples support Sorani, Kurmanji or both")
281
+ return self
282
+
283
+
284
+ class CapacityRequest(BaseModel):
285
+ model_config = ConfigDict(extra="forbid")
286
+
287
+ text: str = Field(default="", max_length=200_000)
288
+ language: str = "sorani"
289
+ languages: LanguageSelection | None = None
290
+
291
+ @model_validator(mode="after")
292
+ def coherent(self):
293
+ if len(self.text.encode("utf-8")) > 200 * 1024:
294
+ raise ValueError("Custom text must be at most 200 KiB")
295
+ if len(_language_tags(self.language, self.languages)) != 1:
296
+ raise ValueError("Choose one explicit source language for your own text")
297
+ return self
298
+
299
+
300
+ def _now():
301
+ return datetime.now(UTC).isoformat()
302
+
303
+
304
+ def _atomic_json(path: Path, value: dict):
305
+ temporary = None
306
+ try:
307
+ with tempfile.NamedTemporaryFile(
308
+ "w", dir=path.parent, encoding="utf-8", delete=False
309
+ ) as file:
310
+ temporary = Path(file.name)
311
+ json.dump(value, file, ensure_ascii=False)
312
+ file.write("\n")
313
+ # Close before renaming: Windows cannot rename a file that is still open.
314
+ os.replace(temporary, path)
315
+ finally:
316
+ if temporary is not None:
317
+ temporary.unlink(missing_ok=True)
318
+
319
+
320
+ def _language_tags(language, languages=None):
321
+ if languages is not None:
322
+ if (
323
+ not isinstance(languages, list)
324
+ or not 1 <= len(languages) <= 4
325
+ or any(tag not in FOCUSED_LANGUAGES for tag in languages)
326
+ or len(set(languages)) != len(languages)
327
+ ):
328
+ raise ValueError(
329
+ "Choose distinct source languages from Sorani, Kurmanji, Arabic and English"
330
+ )
331
+ return list(languages)
332
+ if not isinstance(language, str) or (
333
+ language not in LANGUAGES and not LANGUAGE.fullmatch(language)
334
+ ):
335
+ raise ValueError("Choose a valid source language")
336
+ return [canonical_language(tag) for tag in LANGUAGES.get(language, [language])]
337
+
338
+
339
+ def _custom_source(raw: Path, text: str, language: str) -> Path:
340
+ raw.mkdir()
341
+ path = raw / "custom.jsonl"
342
+ with path.open("x", encoding="utf-8") as stream:
343
+ for line, value in enumerate(io.StringIO(text), 1):
344
+ value = value.removesuffix("\n").removesuffix("\r")
345
+ if not value.strip():
346
+ continue
347
+ stream.write(
348
+ json.dumps(
349
+ {
350
+ "text": value,
351
+ "language": language,
352
+ "genre": "user-text",
353
+ "attribution": {
354
+ "source": "User supplied text",
355
+ "line": line,
356
+ "modified": False,
357
+ },
358
+ },
359
+ ensure_ascii=False,
360
+ )
361
+ + "\n"
362
+ )
363
+ return path
364
+
365
+
366
+ def _lock_output(metadata: Path):
367
+ """Only one studio may recover and write job history for this output root."""
368
+ path = metadata / ".server.lock"
369
+ if path.is_symlink():
370
+ raise ValueError("Studio lock must not be a symbolic link")
371
+ stream = path.open("a+b")
372
+ try:
373
+ if os.name == "nt":
374
+ import msvcrt
375
+
376
+ stream.seek(0)
377
+ if not stream.read(1):
378
+ stream.write(b"0")
379
+ stream.flush()
380
+ stream.seek(0)
381
+ msvcrt.locking(stream.fileno(), msvcrt.LK_NBLCK, 1)
382
+ else:
383
+ import fcntl
384
+
385
+ fcntl.flock(stream.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
386
+ except OSError as error:
387
+ stream.close()
388
+ raise ValueError(
389
+ "Another SynthOCR studio is using this output folder. "
390
+ "Use its browser tab, stop that server, or choose another --output-root."
391
+ ) from error
392
+ return stream
393
+
394
+
395
+ def _archive_size(source: Path):
396
+ """Size of a stored ZIP64, without a dataset-sized entry list."""
397
+ size = 98 # ZIP64 end record, locator and classic end record.
398
+ for path in dataset_files(source):
399
+ if path.is_symlink():
400
+ raise ValueError("Dataset archives cannot contain symbolic links")
401
+ if path.is_file():
402
+ size += (
403
+ path.stat().st_size + 124 + 2 * len(path.relative_to(source).as_posix().encode())
404
+ )
405
+ return size
406
+
407
+
408
+ def _stored_zip(source: Path, destination: Path):
409
+ """Write ZIP64 without ZipFile's in-memory list of every image entry.
410
+
411
+ PNG/JPEG/WebP/TIFF images are already compressed. Stored entries also avoid
412
+ spending CPU recompressing them; the central directory is spooled to disk.
413
+ """
414
+ with destination.open("xb") as output, tempfile.TemporaryFile() as directory:
415
+ count = 0
416
+ for path in dataset_files(source):
417
+ if path.is_symlink():
418
+ raise ValueError("Dataset archives cannot contain symbolic links")
419
+ if not path.is_file():
420
+ continue
421
+ name = path.relative_to(source).as_posix().encode("utf-8")
422
+ size, crc = 0, 0
423
+ with path.open("rb") as stream:
424
+ while chunk := stream.read(1024 * 1024):
425
+ size += len(chunk)
426
+ crc = zlib.crc32(chunk, crc)
427
+ offset = output.tell()
428
+ extra = struct.pack("<HHQQ", 1, 16, size, size)
429
+ output.write(
430
+ struct.pack(
431
+ "<IHHHHHIIIHH",
432
+ 0x04034B50,
433
+ 45,
434
+ 0x800,
435
+ 0,
436
+ 0,
437
+ 33,
438
+ crc,
439
+ 0xFFFFFFFF,
440
+ 0xFFFFFFFF,
441
+ len(name),
442
+ len(extra),
443
+ )
444
+ + name
445
+ + extra
446
+ )
447
+ with path.open("rb") as stream:
448
+ shutil.copyfileobj(stream, output, 1024 * 1024)
449
+ extra = struct.pack("<HHQQQ", 1, 24, size, size, offset)
450
+ directory.write(
451
+ struct.pack(
452
+ "<IHHHHHHIIIHHHHHII",
453
+ 0x02014B50,
454
+ 45,
455
+ 45,
456
+ 0x800,
457
+ 0,
458
+ 0,
459
+ 33,
460
+ crc,
461
+ 0xFFFFFFFF,
462
+ 0xFFFFFFFF,
463
+ len(name),
464
+ len(extra),
465
+ 0,
466
+ 0,
467
+ 0,
468
+ 0,
469
+ 0xFFFFFFFF,
470
+ )
471
+ + name
472
+ + extra
473
+ )
474
+ count += 1
475
+ central_offset, central_size = output.tell(), directory.tell()
476
+ directory.seek(0)
477
+ shutil.copyfileobj(directory, output, 1024 * 1024)
478
+ zip64_offset = output.tell()
479
+ output.write(
480
+ struct.pack(
481
+ "<IQHHIIQQQQ",
482
+ 0x06064B50,
483
+ 44,
484
+ 45,
485
+ 45,
486
+ 0,
487
+ 0,
488
+ count,
489
+ count,
490
+ central_size,
491
+ central_offset,
492
+ )
493
+ )
494
+ output.write(struct.pack("<IIQI", 0x07064B50, 0, zip64_offset, 1))
495
+ output.write(
496
+ struct.pack("<IHHHHIIH", 0x06054B50, 0, 0, 0xFFFF, 0xFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0)
497
+ )
498
+
499
+
500
+ class _ArchiveResponse(FileResponse):
501
+ def __init__(self, path, *, release, **kwargs):
502
+ super().__init__(path, **kwargs)
503
+ self.release = release
504
+
505
+ async def __call__(self, scope, receive, send):
506
+ try:
507
+ await super().__call__(scope, receive, send)
508
+ finally:
509
+ # Also release pins when a browser disconnects partway through a ZIP.
510
+ with CancelScope(shield=True):
511
+ await run_in_threadpool(self.release, self.path)
512
+
513
+
514
+ class Studio:
515
+ def __init__(
516
+ self,
517
+ config: Config,
518
+ output_root: Path | None = None,
519
+ *,
520
+ cache_mb: int = 256,
521
+ output_locations: dict[str, Path] | None = None,
522
+ ):
523
+ if isinstance(cache_mb, bool) or not isinstance(cache_mb, int) or cache_mb < 1:
524
+ raise ValueError("Studio cache size must be a positive number of MiB")
525
+ self.cache_limit = cache_mb * 1024 * 1024
526
+ self.config = config.validate()
527
+ self.output_root = (output_root or config.path("dataset/ui")).resolve()
528
+ self.protected = (
529
+ config.path(config.data.raw),
530
+ config.path(config.data.processed),
531
+ *(config.path(path) for path in config.fonts.directories),
532
+ )
533
+ if output_locations is not None and (
534
+ not isinstance(output_locations, dict)
535
+ or len(output_locations) > 8
536
+ or any(
537
+ not isinstance(name, str) or not LOCATION.fullmatch(name) or name == "default"
538
+ for name in output_locations
539
+ )
540
+ or any(
541
+ not isinstance(path, (str, os.PathLike)) or not str(path).strip()
542
+ for path in output_locations.values()
543
+ )
544
+ ):
545
+ raise ValueError("Use at most eight named output locations; 'default' is reserved")
546
+ self.output_locations = {
547
+ "default": self.output_root,
548
+ **{name: config.path(str(path)) for name, path in (output_locations or {}).items()},
549
+ }
550
+ roots = list(self.output_locations.values())
551
+ for index, root in enumerate(roots):
552
+ self._validate_output_root(root, roots[:index])
553
+ self.metadata = self.output_root / ".jobs"
554
+ self.cache = self.metadata / ".cache"
555
+ self.uploads = self.metadata / "uploads"
556
+ # Fonts added in the app stay in this workspace, never in the project's fonts/.
557
+ self.user_fonts = self.metadata / "fonts"
558
+ self._root_locks = []
559
+ try:
560
+ for root in roots:
561
+ root.mkdir(parents=True, exist_ok=True)
562
+ metadata = root / ".jobs"
563
+ if metadata.is_symlink():
564
+ raise ValueError("Job metadata directory must not be a symbolic link")
565
+ metadata.mkdir(exist_ok=True)
566
+ self._root_locks.append(_lock_output(metadata))
567
+ for path, label in (
568
+ (self.cache, "Studio cache"),
569
+ (self.uploads, "Upload directory"),
570
+ (self.user_fonts, "User font directory"),
571
+ ):
572
+ if path.is_symlink():
573
+ raise ValueError(f"{label} must not be a symbolic link")
574
+ path.mkdir(exist_ok=True)
575
+ except BaseException:
576
+ for lock in reversed(self._root_locks):
577
+ lock.close()
578
+ raise
579
+ self.lock = threading.RLock()
580
+ self.upload_lock = threading.RLock()
581
+ self.upload_validation_lock = threading.Lock()
582
+ self.archive_lock = threading.RLock()
583
+ self.preview_lock = threading.Lock()
584
+ self.executor = ThreadPoolExecutor(max_workers=1, thread_name_prefix="synthocr-studio")
585
+ self.active: dict | None = None
586
+ self.cancel_event = threading.Event()
587
+ self._fonts = None
588
+ self._font_catalog = None
589
+ self._coverage = Coverage(min(config.fonts.coverage_cache_size, 128))
590
+ self._capacities = OrderedDict()
591
+ self._leased: dict[Path, int] = {}
592
+ self._uploading: set[str] = set()
593
+ self._claimed_uploads: set[str] = set()
594
+ self._used_uploads = OrderedDict()
595
+ try:
596
+ self._trim_uploads()
597
+ self._discard_legacy_cache()
598
+ self._trim_cache()
599
+ # Recover at most the displayed history, without loading a dataset-size index.
600
+ for path in self._recent_paths():
601
+ try:
602
+ job = self._load(path)
603
+ self.job_root(job)
604
+ if (
605
+ isinstance(job.get("upload"), dict)
606
+ and (self.dataset(job) / "source-upload.json").is_file()
607
+ ):
608
+ self._used_uploads[job["upload"]["id"]] = job["name"]
609
+ except OSError, ValueError, HTTPException:
610
+ continue
611
+ if job.get("status") in ACTIVE:
612
+ job.update(
613
+ status="failed",
614
+ error="The app closed before generation finished. Start a new run.",
615
+ )
616
+ self._save(job)
617
+ except BaseException:
618
+ self.close()
619
+ raise
620
+
621
+ def _validate_output_root(self, root: Path, others):
622
+ if any(
623
+ path.name.casefold() == ".jobs" or self._is_dataset(path)
624
+ for path in (root, *root.parents)
625
+ ):
626
+ raise ValueError("Choose an output root outside existing datasets and job metadata")
627
+ if any(self._roots_overlap(root, path) for path in self.protected):
628
+ raise ValueError("UI output root must be separate from raw, processed data and fonts")
629
+ if any(self._roots_overlap(root, other) for other in others):
630
+ raise ValueError("Output locations must not overlap or refer to the same folder")
631
+
632
+ @staticmethod
633
+ def _roots_overlap(first: Path, second: Path) -> bool:
634
+ if first.is_relative_to(second) or second.is_relative_to(first):
635
+ return True
636
+ # Case-insensitive volumes may expose the same folder through differently
637
+ # cased POSIX paths. Compare existing ancestors by filesystem identity.
638
+ for root, other in ((first, second), (second, first)):
639
+ if not root.exists():
640
+ continue
641
+ for candidate in (other, *other.parents):
642
+ if candidate.exists() and root.samefile(candidate):
643
+ return True
644
+ return False
645
+
646
+ def _discard_legacy_cache(self):
647
+ # Earlier studio versions retained a second export tree and ZIP for every
648
+ # format. These directories contain only disposable studio derivatives.
649
+ with os.scandir(self.metadata) as entries:
650
+ for entry in entries:
651
+ if JOB_ID.fullmatch(entry.name) and entry.is_dir(follow_symlinks=False):
652
+ shutil.rmtree(entry.path)
653
+ elif entry.name.startswith(".export-") and entry.is_dir(follow_symlinks=False):
654
+ shutil.rmtree(entry.path)
655
+ with os.scandir(self.cache) as entries:
656
+ for entry in entries:
657
+ if entry.name.endswith(".tmp") and entry.is_file(follow_symlinks=False):
658
+ Path(entry.path).unlink(missing_ok=True)
659
+
660
+ def _cache_files(self):
661
+ if self.cache.is_symlink():
662
+ raise HTTPException(400, "Invalid studio cache")
663
+ with os.scandir(self.cache) as entries:
664
+ for entry in entries:
665
+ if entry.is_file(follow_symlinks=False):
666
+ yield Path(entry.path)
667
+
668
+ def _trim_cache(self, reserve=0):
669
+ """Evict only disposable derivatives; never remove original datasets."""
670
+ if reserve > self.cache_limit:
671
+ raise HTTPException(
672
+ 413,
673
+ "This download exceeds the studio cache limit. Download the manifest, "
674
+ "use the dataset in the output folder, or restart with a larger --cache-mb.",
675
+ )
676
+ while True:
677
+ total, count, oldest = 0, 0, None
678
+ for path in self._cache_files():
679
+ info = path.stat()
680
+ total += info.st_size
681
+ count += 1
682
+ if path not in self._leased and (oldest is None or info.st_mtime_ns < oldest[0]):
683
+ oldest = (info.st_mtime_ns, path)
684
+ if total + reserve <= self.cache_limit and count < CACHE_FILES:
685
+ return total
686
+ if oldest is None:
687
+ raise HTTPException(
688
+ 503, "Download cache is busy; retry after current downloads finish"
689
+ )
690
+ oldest[1].unlink(missing_ok=True)
691
+
692
+ def storage(self):
693
+ with self.archive_lock:
694
+ return {
695
+ "cache_limit_bytes": self.cache_limit,
696
+ "current_cache_bytes": sum(path.stat().st_size for path in self._cache_files()),
697
+ "policy": "Least recently used downloads are removed; datasets are retained",
698
+ "upload_limit_bytes": UPLOAD_BYTES,
699
+ }
700
+
701
+ def location_root(self, location: str, expected: str | None = None) -> Path:
702
+ if not isinstance(location, str) or location not in self.output_locations:
703
+ raise HTTPException(404, "Output location is not available")
704
+ root = self.output_locations[location]
705
+ if expected is not None and expected != str(root):
706
+ raise HTTPException(
707
+ 409, "This dataset's output location has changed; restore its original location"
708
+ )
709
+ if root.is_symlink() or root.resolve() != root or not root.is_dir():
710
+ raise HTTPException(400, "Output location is not an accessible ordinary directory")
711
+ return root
712
+
713
+ def job_root(self, job) -> Path:
714
+ location = job.get("output_location", "default")
715
+ expected = job.get("output_root")
716
+ if location != "default" and not isinstance(expected, str):
717
+ raise HTTPException(400, "Dataset output location identity is missing")
718
+ return self.location_root(location, expected)
719
+
720
+ @staticmethod
721
+ def _is_dataset(path: Path) -> bool:
722
+ return any(
723
+ (path / name).exists() or (path / name).is_symlink()
724
+ for name in ("run.json", "manifest.jsonl")
725
+ )
726
+
727
+ def directory(self, location: str, path: str = "", expected: str | None = None) -> Path:
728
+ root = self.location_root(location, expected)
729
+ parts = _folder_parts(path)
730
+ # Generation creates the image directory before writing run/manifest
731
+ # markers. Reserve the entire active destination during that interval,
732
+ # including queued output that does not yet exist. Derive its path from
733
+ # recorded components; dataset() would recurse through directory().
734
+ reserved = None
735
+ active = self.active
736
+ if active and active["status"] in ACTIVE:
737
+ active_root = self.output_locations.get(active.get("output_location", "default"))
738
+ name = active.get("name")
739
+ if (
740
+ active_root
741
+ and isinstance(name, str)
742
+ and NAME.fullmatch(name)
743
+ and _folder_name(name)
744
+ ):
745
+ reserved = active_root.joinpath(
746
+ *_folder_parts(active.get("output_folder", "")), name
747
+ )
748
+ current = root
749
+ for index in range(len(parts) + 1):
750
+ if reserved is not None and current.is_relative_to(reserved):
751
+ raise HTTPException(409, "That folder is reserved for the active dataset")
752
+ if current.is_symlink() or current.resolve() != current or not current.is_dir():
753
+ raise HTTPException(400, "Choose an existing folder without symbolic links")
754
+ if self._is_dataset(current):
755
+ raise HTTPException(400, "Choose a folder outside existing datasets")
756
+ if any(current.is_relative_to(protected) for protected in self.protected):
757
+ raise HTTPException(
758
+ 400, "Output folders must stay separate from source data and fonts"
759
+ )
760
+ if index < len(parts):
761
+ current = current / parts[index]
762
+ return current
763
+
764
+ def directories(self, location: str = "default", path: str = ""):
765
+ parent = self.directory(location, path)
766
+ children, truncated = [], False
767
+ with os.scandir(parent) as entries:
768
+ for scanned, entry in enumerate(entries):
769
+ if scanned >= DIRECTORY_SCAN_LIMIT:
770
+ truncated = True
771
+ break
772
+ if not _folder_name(entry.name) or not entry.is_dir(follow_symlinks=False):
773
+ continue
774
+ relative = "/".join(filter(None, (path, entry.name)))
775
+ try:
776
+ self.directory(location, relative)
777
+ except OSError, ValueError, HTTPException:
778
+ continue
779
+ if len(children) >= DIRECTORY_LIMIT:
780
+ truncated = True
781
+ break
782
+ children.append({"name": entry.name, "path": relative})
783
+ children.sort(key=lambda entry: entry["name"].casefold())
784
+ return {
785
+ "location": location,
786
+ "path": path,
787
+ "parent": path.rpartition("/")[0] if path else None,
788
+ "directories": children,
789
+ "truncated": truncated,
790
+ "root": str(self.output_locations[location]),
791
+ "absolute_path": str(parent),
792
+ }
793
+
794
+ def create_directory(self, request: DirectoryRequest):
795
+ with self.lock:
796
+ relative = "/".join(filter(None, (request.path, request.name)))
797
+ _folder_parts(relative)
798
+ parent = self.directory(request.location, request.path)
799
+ target = parent / request.name
800
+ if target.exists() or target.is_symlink():
801
+ raise HTTPException(
802
+ 409, "That folder already exists; choose it or use another name"
803
+ )
804
+ if (
805
+ self.active
806
+ and self.active["status"] in ACTIVE
807
+ and target == self.dataset(self.active)
808
+ ):
809
+ raise HTTPException(409, "That folder is reserved for the active dataset")
810
+ target.mkdir()
811
+ return self.directories(request.location, relative)
812
+
813
+ def _pending_uploads(self):
814
+ if self.uploads.is_symlink():
815
+ raise HTTPException(400, "Invalid upload directory")
816
+ with os.scandir(self.uploads) as entries:
817
+ for entry in entries:
818
+ if JOB_ID.fullmatch(entry.name) and entry.is_dir(follow_symlinks=False):
819
+ yield Path(entry.path)
820
+
821
+ def _trim_uploads(self):
822
+ """Expire unused inputs only; retained dataset sources are never removed."""
823
+ with self.upload_lock:
824
+ now = time.time()
825
+ for path in self._pending_uploads():
826
+ if path.name in self._uploading | self._claimed_uploads:
827
+ continue
828
+ if not (path / "upload.json").is_file() or now - path.stat().st_mtime > UPLOAD_TTL:
829
+ shutil.rmtree(path)
830
+
831
+ def _upload_usage(self):
832
+ size, count = 0, 0
833
+ for path in self._pending_uploads():
834
+ count += 1
835
+ with os.scandir(path) as entries:
836
+ for entry in entries:
837
+ if entry.name.startswith("source.") and entry.is_file(follow_symlinks=False):
838
+ size += entry.stat(follow_symlinks=False).st_size
839
+ return size, count
840
+
841
+ def _begin_upload(self, filename, language):
842
+ if (
843
+ not filename
844
+ or len(filename.encode("utf-8")) > 255
845
+ or Path(filename).name != filename
846
+ or "\\" in filename
847
+ or any(ord(char) < 32 for char in filename)
848
+ ):
849
+ raise HTTPException(422, "Choose a TXT, JSONL or CSV file")
850
+ suffix = Path(filename).suffix.lower()
851
+ if suffix not in {".txt", ".jsonl", ".csv"}:
852
+ raise HTTPException(422, "Upload a UTF-8 TXT, JSONL or CSV file")
853
+ if len(language) > 128 or (language not in LANGUAGES and not LANGUAGE.fullmatch(language)):
854
+ raise HTTPException(422, "Choose a source language for the uploaded file")
855
+ with self.upload_lock:
856
+ self._trim_uploads()
857
+ size, count = self._upload_usage()
858
+ if count >= UPLOAD_FILES or size >= UPLOAD_TOTAL_BYTES:
859
+ raise HTTPException(
860
+ 413, "Pending uploads are full. Remove an unused upload or generate its dataset"
861
+ )
862
+ identifier = uuid4().hex
863
+ path = self.uploads / identifier
864
+ path.mkdir()
865
+ self._uploading.add(identifier)
866
+ return identifier, path / f"source{suffix}"
867
+
868
+ def _write_upload_chunk(self, stream, chunk):
869
+ with self.upload_lock:
870
+ if stream.tell() + len(chunk) > UPLOAD_BYTES:
871
+ raise HTTPException(413, "Uploaded files must be at most 50 MiB")
872
+ size, _ = self._upload_usage()
873
+ if size + len(chunk) > UPLOAD_TOTAL_BYTES:
874
+ raise HTTPException(413, "Pending uploads exceed the 100 MiB storage limit")
875
+ stream.write(chunk)
876
+ # Make accepted bytes visible to quota checks for simultaneous uploads.
877
+ stream.flush()
878
+
879
+ def _validate_upload(self, identifier, path, filename, language, selected_languages=None):
880
+ with (
881
+ self.upload_validation_lock,
882
+ tempfile.TemporaryDirectory(prefix="synthocr-upload-index-") as directory,
883
+ ):
884
+ db = sqlite3.connect(Path(directory) / "labels.sqlite3")
885
+ try:
886
+ db.execute("PRAGMA cache_size=-2048")
887
+ db.execute("PRAGMA temp_store=FILE")
888
+ db.execute("PRAGMA journal_mode=OFF")
889
+ db.execute(
890
+ "CREATE TABLE labels (id INTEGER PRIMARY KEY, language TEXT, text TEXT, "
891
+ "genre TEXT, UNIQUE(language,text))"
892
+ )
893
+ return self._inspect_upload(
894
+ identifier, path, filename, language, db, selected_languages
895
+ )
896
+ finally:
897
+ db.close()
898
+
899
+ def _inspect_upload(self, identifier, path, filename, language, db, selected_languages=None):
900
+ selection_languages = _language_tags(language, selected_languages)
901
+ selected = set(selection_languages)
902
+ excluded = {*self.config.generation.exclude_genres, "spelling"}
903
+ fallback = selection_languages
904
+ fallback = fallback[0] if len(fallback) == 1 else ""
905
+ count, unlabelled, languages = 0, 0, set()
906
+ for line, row in records(path, path.parent, self.config.data.max_line_chars):
907
+ value = row.get("text")
908
+ if not isinstance(value, str):
909
+ raise ValueError(f"Line {line}: text must be a string")
910
+ value.encode("utf-8", errors="strict")
911
+ text = normalize(value, "NFC")
912
+ if not required_codepoints(text):
913
+ continue
914
+ tag = row.get("language") or ""
915
+ if not isinstance(tag, str) or len(tag) > 128:
916
+ raise ValueError(f"Line {line}: language must be a language tag")
917
+ if not isinstance(row.get("attribution", {}), dict):
918
+ raise ValueError(f"Line {line}: attribution must be an object")
919
+ genre = row.get("genre") or path.stem
920
+ if not isinstance(genre, str):
921
+ raise ValueError(f"Line {line}: genre must be a string")
922
+ if not tag:
923
+ unlabelled += 1
924
+ if not fallback:
925
+ raise HTTPException(
926
+ 422,
927
+ "This file has text without language tags. Choose one explicit language, "
928
+ "or add a language field to every JSONL/CSV record",
929
+ )
930
+ tag = fallback
931
+ tag = resolve_source_language(tag, row.get("attribution", {}))
932
+ tag = canonical_language(tag)
933
+ languages.add(tag)
934
+ if len(languages) > 128:
935
+ raise HTTPException(422, "An uploaded file may contain at most 128 language tags")
936
+ count += 1
937
+ for label in chunks(text, self.config.data.max_chars):
938
+ if required_codepoints(label):
939
+ db.execute(
940
+ "INSERT OR IGNORE INTO labels(language,text,genre) VALUES(?,?,?)",
941
+ (tag, label, genre),
942
+ )
943
+ # Shared preparation keeps the first origin, except that
944
+ # natural text replaces an identical spelling diagnostic.
945
+ if genre != "spelling":
946
+ db.execute(
947
+ "UPDATE labels SET genre=? WHERE language=? AND text=? "
948
+ "AND genre='spelling'",
949
+ (genre, tag, label),
950
+ )
951
+ if not count:
952
+ raise HTTPException(422, "The file contains no visible text or numbers")
953
+ conditions, parameters = [], []
954
+ if selected:
955
+ conditions.append(f"language IN ({','.join('?' for _ in selected)})")
956
+ parameters.extend(sorted(selected))
957
+ if excluded:
958
+ conditions.append(f"genre NOT IN ({','.join('?' for _ in excluded)})")
959
+ parameters.extend(sorted(excluded))
960
+ selection = " WHERE " + " AND ".join(conditions) if conditions else ""
961
+ selected_records = db.execute(
962
+ "SELECT COUNT(*) FROM labels" + selection, parameters
963
+ ).fetchone()[0]
964
+ if not selected_records:
965
+ raise HTTPException(
966
+ 422,
967
+ "The uploaded file has no usable text for the selected language. "
968
+ "Check its language tags and excluded genres",
969
+ )
970
+ preview_language, first_label = db.execute(
971
+ "SELECT language,text FROM labels" + selection + " ORDER BY id LIMIT 1", parameters
972
+ ).fetchone()
973
+ preview = next(chunks(first_label, min(self.config.data.max_chars, 200)), "")
974
+ document = {
975
+ "id": identifier,
976
+ "filename": filename,
977
+ "stored_filename": path.name,
978
+ "bytes": path.stat().st_size,
979
+ "sha256": sha256(path),
980
+ "records": selected_records,
981
+ "source_records": count,
982
+ "languages": sorted(languages),
983
+ "selected_languages": selection_languages,
984
+ "unlabelled_records": unlabelled,
985
+ "language": language,
986
+ "preview_text": preview,
987
+ "preview_language": preview_language,
988
+ "created_at": _now(),
989
+ }
990
+ _atomic_json(path.parent / "upload.json", document)
991
+ return document
992
+
993
+ async def upload(self, request, filename, language, languages=None):
994
+ try:
995
+ _language_tags(language, languages)
996
+ except ValueError as error:
997
+ raise HTTPException(422, str(error)) from error
998
+ identifier, path = await run_in_threadpool(self._begin_upload, filename, language)
999
+ successful = False
1000
+ try:
1001
+ with path.open("xb", buffering=0) as stream:
1002
+ async for chunk in request.stream():
1003
+ # ASGI controls incoming chunk size; disk writes and quota checks
1004
+ # stay bounded even when a client sends the entire body at once.
1005
+ view = memoryview(chunk)
1006
+ for start in range(0, len(view), UPLOAD_CHUNK):
1007
+ await run_in_threadpool(
1008
+ self._write_upload_chunk, stream, view[start : start + UPLOAD_CHUNK]
1009
+ )
1010
+ arguments = (identifier, path, filename, language)
1011
+ result = await run_in_threadpool(
1012
+ self._validate_upload, *arguments, *([languages] if languages is not None else [])
1013
+ )
1014
+ if await request.is_disconnected():
1015
+ raise HTTPException(499, "Upload cancelled")
1016
+ successful = True
1017
+ return result
1018
+ finally:
1019
+ with self.upload_lock:
1020
+ self._uploading.discard(identifier)
1021
+ if not successful:
1022
+ shutil.rmtree(path.parent, ignore_errors=True)
1023
+
1024
+ def uploaded(self, identifier):
1025
+ if not JOB_ID.fullmatch(identifier):
1026
+ raise HTTPException(404, "Upload not found; choose the file again")
1027
+ with self.upload_lock:
1028
+ if identifier in self._used_uploads:
1029
+ raise HTTPException(
1030
+ 409,
1031
+ f"This file was saved with dataset {self._used_uploads[identifier]}. "
1032
+ "Choose the file again to generate another dataset",
1033
+ )
1034
+ self._trim_uploads()
1035
+ directory = self.uploads / identifier
1036
+ metadata = directory / "upload.json"
1037
+ try:
1038
+ if (
1039
+ directory.is_symlink()
1040
+ or metadata.is_symlink()
1041
+ or metadata.stat().st_size > 65536
1042
+ ):
1043
+ raise ValueError("Invalid upload metadata")
1044
+ upload = json.loads(metadata.read_text(encoding="utf-8"))
1045
+ filename = upload["stored_filename"]
1046
+ if filename not in {"source.txt", "source.csv", "source.jsonl"}:
1047
+ raise ValueError("Invalid upload file")
1048
+ path = directory / filename
1049
+ if path.is_symlink() or not path.is_file() or upload["id"] != identifier:
1050
+ raise ValueError("Invalid upload source")
1051
+ except (OSError, ValueError, TypeError, KeyError) as error:
1052
+ raise HTTPException(404, "Upload not found; choose the file again") from error
1053
+ return upload, path
1054
+
1055
+ def remove_upload(self, identifier):
1056
+ with self.upload_lock:
1057
+ self.uploaded(identifier)
1058
+ if identifier in self._claimed_uploads:
1059
+ raise HTTPException(409, "This file is being used by the current generation")
1060
+ shutil.rmtree(self.uploads / identifier)
1061
+ return {"removed": True}
1062
+
1063
+ def _release_archive(self, path):
1064
+ with self.archive_lock:
1065
+ self._leased[path] -= 1
1066
+ if not self._leased[path]:
1067
+ del self._leased[path]
1068
+
1069
+ def _recent_paths(self):
1070
+ def candidates():
1071
+ with os.scandir(self.metadata) as entries:
1072
+ for entry in entries:
1073
+ if (
1074
+ entry.name.endswith(".json")
1075
+ and JOB_ID.fullmatch(entry.name[:-5])
1076
+ and entry.is_file(follow_symlinks=False)
1077
+ ):
1078
+ yield Path(entry.path)
1079
+
1080
+ return heapq.nlargest(
1081
+ 20,
1082
+ candidates(),
1083
+ key=lambda p: p.stat().st_mtime_ns,
1084
+ )
1085
+
1086
+ def _load(self, path):
1087
+ if path.is_symlink() or path.stat().st_size > 64 * 1024:
1088
+ raise ValueError("Invalid job metadata")
1089
+ job = json.loads(path.read_text(encoding="utf-8"))
1090
+ if (
1091
+ not isinstance(job, dict)
1092
+ or job.get("id") != path.stem
1093
+ or not isinstance(job.get("name"), str)
1094
+ or not NAME.fullmatch(job["name"])
1095
+ or not isinstance(job.get("status"), str)
1096
+ or job.get("status") not in ACTIVE | {"completed", "cancelled", "failed"}
1097
+ ):
1098
+ raise ValueError("Invalid job metadata")
1099
+ return job
1100
+
1101
+ def _save(self, job):
1102
+ job["updated_at"] = _now()
1103
+ _atomic_json(self.metadata / f"{job['id']}.json", job)
1104
+
1105
+ def job(self, identifier):
1106
+ if not JOB_ID.fullmatch(identifier):
1107
+ raise HTTPException(404, "Job not found")
1108
+ with self.lock:
1109
+ if self.active and self.active["id"] == identifier:
1110
+ return self.active.copy()
1111
+ path = self.metadata / f"{identifier}.json"
1112
+ try:
1113
+ job = self._load(path)
1114
+ except (OSError, ValueError, KeyError) as error:
1115
+ raise HTTPException(404, "Job not found") from error
1116
+ if job.get("id") != identifier or not NAME.fullmatch(job.get("name", "")):
1117
+ raise HTTPException(404, "Job not found")
1118
+ self.job_root(job)
1119
+ if job.get("status") in ACTIVE:
1120
+ job.update(
1121
+ status="failed",
1122
+ error="The app closed before generation finished. Start a new run.",
1123
+ )
1124
+ self._save(job)
1125
+ return job
1126
+
1127
+ def jobs(self):
1128
+ with self.lock:
1129
+ result = []
1130
+ for path in self._recent_paths():
1131
+ try:
1132
+ result.append(self.job(path.stem))
1133
+ except HTTPException:
1134
+ continue
1135
+ return result
1136
+
1137
+ def dataset(self, job):
1138
+ root = self.job_root(job)
1139
+ parent = self.directory(
1140
+ job.get("output_location", "default"), job.get("output_folder", ""), str(root)
1141
+ )
1142
+ if not NAME.fullmatch(job["name"]) or not _folder_name(job["name"]):
1143
+ raise HTTPException(400, "Invalid dataset name")
1144
+ path = parent / job["name"]
1145
+ if path.is_symlink() or path.resolve().parent != parent:
1146
+ raise HTTPException(400, "Dataset path must stay inside the configured output root")
1147
+ if any(
1148
+ (path / name).is_symlink()
1149
+ for name in (
1150
+ "manifest.jsonl",
1151
+ "origins.jsonl",
1152
+ "run.json",
1153
+ "stats.json",
1154
+ "images",
1155
+ "source.jsonl",
1156
+ "source.txt",
1157
+ "source.csv",
1158
+ "source-upload.json",
1159
+ )
1160
+ ):
1161
+ raise HTTPException(400, "Dataset artifacts must not be symbolic links")
1162
+ return path
1163
+
1164
+ def font_directories(self):
1165
+ return [*(self.config.path(p) for p in self.config.fonts.directories), self.user_fonts]
1166
+
1167
+ def fonts(self):
1168
+ with self.lock:
1169
+ if self._fonts is None:
1170
+ self._fonts = discover(self.font_directories())
1171
+ return self._fonts
1172
+
1173
+ def font_catalog(self):
1174
+ """Every usable face with a readable name and the page languages it draws."""
1175
+ with self.lock:
1176
+ if self._font_catalog is None:
1177
+ fonts = self.fonts()
1178
+ ids = font_ids(fonts)
1179
+ user = self.user_fonts.resolve()
1180
+ entries = [
1181
+ {
1182
+ "id": ids[font],
1183
+ "name": _font_display_name(font),
1184
+ "file": Path(font.path).name,
1185
+ "user": Path(font.path).parent == user,
1186
+ "languages": [
1187
+ tag
1188
+ for tag, sample in LANGUAGE_SAMPLES.items()
1189
+ if not self._coverage.missing(font, required_codepoints(sample))
1190
+ ],
1191
+ }
1192
+ for font in fonts
1193
+ ]
1194
+ entries.sort(key=lambda entry: (entry["name"].casefold(), entry["id"]))
1195
+ self._font_catalog = (entries, ids)
1196
+ return self._font_catalog
1197
+
1198
+ def selected_fonts(self, include):
1199
+ fonts = self.fonts()
1200
+ if not include:
1201
+ return fonts
1202
+ try:
1203
+ return select(fonts, include, self.font_catalog()[1])
1204
+ except ValueError as error:
1205
+ raise HTTPException(422, str(error)) from error
1206
+
1207
+ def _forget_fonts(self):
1208
+ self._fonts = None
1209
+ self._font_catalog = None
1210
+
1211
+ async def add_font(self, request, filename):
1212
+ suffix = Path(filename).suffix.lower()
1213
+ if suffix not in FONT_SUFFIXES or not _folder_name(Path(filename).name):
1214
+ raise HTTPException(422, "Choose a TTF, OTF, TTC or OTC font file")
1215
+ # Not a font suffix, so discovery never sees a partial upload.
1216
+ temporary = self.user_fonts / f".upload-{uuid4().hex}.part"
1217
+ try:
1218
+ size = 0
1219
+ with temporary.open("xb") as stream:
1220
+ async for chunk in request.stream():
1221
+ size += len(chunk)
1222
+ if size > FONT_UPLOAD_BYTES:
1223
+ raise HTTPException(413, "Font files must be at most 32 MiB")
1224
+ await run_in_threadpool(stream.write, chunk)
1225
+ return await run_in_threadpool(self._install_font, temporary, filename, suffix)
1226
+ finally:
1227
+ temporary.unlink(missing_ok=True)
1228
+
1229
+ def _install_font(self, temporary, filename, suffix):
1230
+ if not temporary.stat().st_size:
1231
+ raise HTTPException(422, "The font file is empty")
1232
+ try:
1233
+ faces = 1
1234
+ if suffix in {".ttc", ".otc"}:
1235
+ with TTCollection(temporary, lazy=True) as collection:
1236
+ faces = len(collection.fonts)
1237
+ for index in range(faces):
1238
+ ImageFont.truetype(str(temporary), 12, index=index)
1239
+ except (OSError, ValueError, TTLibError) as error:
1240
+ raise HTTPException(422, "This file is not a usable font") from error
1241
+ digest = sha256(temporary)
1242
+ with self.lock:
1243
+ entries, _ = self.font_catalog()
1244
+ existing = [entry for entry in entries if entry["id"].startswith(f"{digest}:")]
1245
+ if existing:
1246
+ return {"fonts": existing, "added": False}
1247
+ if sum(1 for _ in self.user_fonts.glob("*")) >= USER_FONT_LIMIT:
1248
+ raise HTTPException(409, f"At most {USER_FONT_LIMIT} fonts can be added")
1249
+ stem = re.sub(r"[^A-Za-z0-9._-]+", "-", Path(filename).stem).strip("-.")[:60]
1250
+ os.replace(temporary, self.user_fonts / f"{stem or 'font'}-{digest[:12]}{suffix}")
1251
+ self._forget_fonts()
1252
+ entries, _ = self.font_catalog()
1253
+ added = [entry for entry in entries if entry["id"].startswith(f"{digest}:")]
1254
+ return {"fonts": added, "added": True}
1255
+
1256
+ def remove_font(self, digest):
1257
+ with self.lock:
1258
+ if self.active and self.active["status"] in ACTIVE:
1259
+ raise HTTPException(409, "Wait for the current generation to finish")
1260
+ entries, _ = self.font_catalog()
1261
+ match = next(
1262
+ (e for e in entries if e["user"] and e["id"].startswith(f"{digest}:")), None
1263
+ )
1264
+ if match is None:
1265
+ raise HTTPException(404, "Only fonts you added can be removed")
1266
+ path = self.user_fonts / match["file"]
1267
+ if path.is_symlink() or path.parent.resolve() != self.user_fonts.resolve():
1268
+ raise HTTPException(400, "Invalid font file")
1269
+ path.unlink()
1270
+ self._forget_fonts()
1271
+ return {"removed": digest}
1272
+
1273
+ def start(self, request: JobRequest):
1274
+ with self.lock:
1275
+ if self.active and self.active["status"] in ACTIVE:
1276
+ raise HTTPException(
1277
+ 409, "A generation is already running; cancel or finish it first"
1278
+ )
1279
+ self.directory(request.output_location, request.output_folder)
1280
+ self.selected_fonts(request.fonts)
1281
+ if request.source == "custom":
1282
+ capacity = self.capacity(
1283
+ CapacityRequest(
1284
+ text=request.text, language=request.language, languages=request.languages
1285
+ )
1286
+ )
1287
+ if capacity["unsupported_texts"]:
1288
+ raise HTTPException(
1289
+ 422,
1290
+ {
1291
+ "message": (
1292
+ "Installed fonts cannot render every custom line; "
1293
+ "edit the text or add compatible fonts"
1294
+ ),
1295
+ **capacity,
1296
+ },
1297
+ )
1298
+ if request.samples > capacity["available_pairs"]:
1299
+ raise HTTPException(
1300
+ 422,
1301
+ {
1302
+ "message": (
1303
+ f"Your {capacity['texts']} distinct lines support "
1304
+ f"{capacity['available_pairs']} unique text/font pairs. "
1305
+ "Request fewer images or add more text"
1306
+ ),
1307
+ **capacity,
1308
+ },
1309
+ )
1310
+ tags = _language_tags(request.language, request.languages)
1311
+ required = (
1312
+ {row["attribution"]["source_id"] for row in spelling_cases() if row["language"] in tags}
1313
+ if request.source == "spelling"
1314
+ else set()
1315
+ )
1316
+ if request.samples < len(required):
1317
+ raise HTTPException(422, f"This spelling check needs at least {len(required)} images")
1318
+ with self.lock:
1319
+ if self.active and self.active["status"] in ACTIVE:
1320
+ raise HTTPException(
1321
+ 409, "A generation is already running; finish or cancel it first"
1322
+ )
1323
+ identifier = uuid4().hex
1324
+ name = request.name or f"{request.language}-{identifier[:8]}"
1325
+ job = {
1326
+ "id": identifier,
1327
+ "name": name,
1328
+ "language": request.language,
1329
+ "languages": tags,
1330
+ "source": request.source,
1331
+ "status": "queued",
1332
+ "phase": "queued",
1333
+ "generated": 0,
1334
+ "requested": request.samples,
1335
+ "progress": 0,
1336
+ "output_location": request.output_location,
1337
+ "output_folder": request.output_folder,
1338
+ "output_root": str(self.location_root(request.output_location)),
1339
+ "error": None,
1340
+ "result": None,
1341
+ "created_at": _now(),
1342
+ "updated_at": _now(),
1343
+ "settings": request.model_dump(exclude={"text"}),
1344
+ }
1345
+ destination = self.dataset(job)
1346
+ job["output"] = str(destination)
1347
+ if destination.exists():
1348
+ raise HTTPException(409, "That dataset name already exists; choose a fresh name")
1349
+ if request.source == "upload":
1350
+ with self.upload_lock:
1351
+ upload, path = self.uploaded(request.upload_id)
1352
+ selected = {canonical_language(tag) for tag in tags}
1353
+ upload_selection = upload.get(
1354
+ "selected_languages", _language_tags(upload["language"])
1355
+ )
1356
+ if upload["unlabelled_records"] and set(tags) != set(upload_selection):
1357
+ raise HTTPException(
1358
+ 422,
1359
+ "This file contains unlabelled text. Keep its selected language, "
1360
+ "or upload it again after changing the language",
1361
+ )
1362
+ if selected and not selected.intersection(upload["languages"]):
1363
+ raise HTTPException(422, "The uploaded file has no text in this language")
1364
+ if sha256(path) != upload["sha256"]:
1365
+ raise HTTPException(422, "The uploaded file changed; upload it again")
1366
+ self._claimed_uploads.add(request.upload_id)
1367
+ job["upload"] = upload
1368
+ self.active = job
1369
+ self.cancel_event = threading.Event()
1370
+ self._save(job)
1371
+ try:
1372
+ self.executor.submit(self._run, job, request, required, self.cancel_event)
1373
+ except BaseException:
1374
+ self._claimed_uploads.discard(request.upload_id)
1375
+ raise
1376
+ return job.copy()
1377
+
1378
+ def cancel(self, identifier):
1379
+ with self.lock:
1380
+ job = self.job(identifier)
1381
+ if self.active and self.active["id"] == identifier and job["status"] in ACTIVE:
1382
+ self.cancel_event.set()
1383
+ self.active["status"] = "cancelling"
1384
+ self.active["phase"] = "Finishing the current operation before stopping"
1385
+ self._save(self.active)
1386
+ return self.active.copy()
1387
+ return job
1388
+
1389
+ def _run(self, job, request, required, cancellation):
1390
+ last_saved = 0
1391
+
1392
+ def update(progress):
1393
+ nonlocal last_saved
1394
+ with self.lock:
1395
+ job["generated"] = progress.get("generated", job["generated"])
1396
+ job["progress"] = round(100 * job["generated"] / job["requested"], 1)
1397
+ if not cancellation.is_set():
1398
+ job["status"] = progress["stage"]
1399
+ job["phase"] = progress["stage"]
1400
+ if time.monotonic() - last_saved >= 0.5 or progress["stage"] == "verifying":
1401
+ self._save(job)
1402
+ last_saved = time.monotonic()
1403
+
1404
+ def stopped():
1405
+ if cancellation.is_set():
1406
+ raise GenerationCancelled("Generation cancelled; partial output has been retained")
1407
+
1408
+ try:
1409
+ tags = _language_tags(request.language, request.languages)
1410
+ max_chars = self.config.data.max_chars
1411
+ if required:
1412
+ max_chars = max(
1413
+ max_chars,
1414
+ max(len(row["text"]) for row in spelling_cases() if row["language"] in tags),
1415
+ )
1416
+ config = replace(
1417
+ self.config,
1418
+ data=replace(self.config.data, normalization="NFC", max_chars=max_chars),
1419
+ fonts=replace(
1420
+ self.config.fonts,
1421
+ directories=[str(path) for path in self.font_directories()],
1422
+ include=request.fonts,
1423
+ coverage="strict",
1424
+ cache_size=min(self.config.fonts.cache_size, 8),
1425
+ coverage_cache_size=min(self.config.fonts.coverage_cache_size, 128),
1426
+ ),
1427
+ generation=replace(
1428
+ self.config.generation,
1429
+ output=str(self.dataset(job)),
1430
+ samples=request.samples,
1431
+ font_size=request.font_size,
1432
+ format=request.format,
1433
+ **{name: getattr(request, name) for name in AUGMENTATION_FIELDS},
1434
+ seed=request.seed,
1435
+ workers=request.workers,
1436
+ train_ratio=request.train_ratio,
1437
+ validation_ratio=request.validation_ratio,
1438
+ languages=tags,
1439
+ language_weights=request.language_weights,
1440
+ language="",
1441
+ direction="auto",
1442
+ suite="kurdish-spelling" if required else "",
1443
+ exclude_genres=[]
1444
+ if required
1445
+ else [*self.config.generation.exclude_genres, "spelling"],
1446
+ ),
1447
+ ).validate()
1448
+ stopped()
1449
+ with tempfile.TemporaryDirectory(prefix="synthocr-studio-") as directory:
1450
+ custom = None
1451
+ uploaded = None
1452
+ if request.source == "custom":
1453
+ raw, processed = Path(directory) / "raw", Path(directory) / "processed"
1454
+ custom = _custom_source(raw, request.text, tags[0])
1455
+ config = replace(
1456
+ config,
1457
+ data=replace(
1458
+ config.data,
1459
+ raw=str(raw),
1460
+ processed=str(processed),
1461
+ source_languages={},
1462
+ deduplicate=True,
1463
+ write_jsonl=False,
1464
+ ),
1465
+ )
1466
+ elif request.source == "upload":
1467
+ upload, source = self.uploaded(request.upload_id)
1468
+ raw, processed = Path(directory) / "raw", Path(directory) / "processed"
1469
+ raw.mkdir()
1470
+ uploaded = raw / source.name
1471
+ shutil.copyfile(source, uploaded)
1472
+ fallback = upload.get("selected_languages", _language_tags(upload["language"]))
1473
+ config = replace(
1474
+ config,
1475
+ data=replace(
1476
+ config.data,
1477
+ raw=str(raw),
1478
+ processed=str(processed),
1479
+ source_languages={uploaded.name: fallback[0]}
1480
+ if len(fallback) == 1
1481
+ else {},
1482
+ deduplicate=True,
1483
+ write_jsonl=False,
1484
+ ),
1485
+ )
1486
+ try:
1487
+ result = generate(config, progress=update, cancel=cancellation.is_set)
1488
+ finally:
1489
+ # Retain user raw data alongside partial outputs as well as completed ones.
1490
+ if custom and self.dataset(job).is_dir():
1491
+ shutil.copyfile(custom, self.dataset(job) / "source.jsonl")
1492
+ (self.dataset(job) / "source.txt").write_text(
1493
+ request.text, encoding="utf-8"
1494
+ )
1495
+ if uploaded and self.dataset(job).is_dir():
1496
+ target = self.dataset(job) / uploaded.name
1497
+ shutil.copyfile(uploaded, target)
1498
+ _atomic_json(self.dataset(job) / "source-upload.json", upload)
1499
+ with self.upload_lock:
1500
+ shutil.rmtree(self.uploads / request.upload_id)
1501
+ self._used_uploads[request.upload_id] = job["name"]
1502
+ if len(self._used_uploads) > 20:
1503
+ self._used_uploads.popitem(last=False)
1504
+ stopped()
1505
+ update({"stage": "verifying"})
1506
+ result["verification"] = verify(self.dataset(job))
1507
+ if required:
1508
+ missing = required.copy()
1509
+ with (self.dataset(job) / "manifest.jsonl").open(encoding="utf-8") as stream:
1510
+ for record in map(json.loads, stream):
1511
+ missing.discard(record["attribution"]["source_id"])
1512
+ if missing:
1513
+ raise ValueError(f"Spelling check missed {len(missing)} cases; inspect fonts")
1514
+ result["spelling_cases_checked"] = len(required)
1515
+ stopped()
1516
+ with self.lock:
1517
+ job.update(status="completed", phase="verified", result=result, progress=100)
1518
+ except GenerationCancelled as error:
1519
+ with self.lock:
1520
+ job.update(status="cancelled", phase="cancelled", error=str(error))
1521
+ except Exception as error:
1522
+ with self.lock:
1523
+ job.update(status="failed", phase="failed", error=str(error))
1524
+ finally:
1525
+ with self.upload_lock:
1526
+ self._claimed_uploads.discard(request.upload_id)
1527
+ with self.lock:
1528
+ self._save(job)
1529
+
1530
+ def close(self):
1531
+ self.cancel_event.set()
1532
+ try:
1533
+ self.executor.shutdown(wait=True, cancel_futures=True)
1534
+ finally:
1535
+ for lock in reversed(self._root_locks):
1536
+ lock.close()
1537
+
1538
+ def capacity(self, request: CapacityRequest):
1539
+ with self.preview_lock:
1540
+ key = hashlib.sha256(
1541
+ (
1542
+ "\0".join(_language_tags(request.language, request.languages))
1543
+ + "\0"
1544
+ + request.text
1545
+ ).encode("utf-8")
1546
+ ).hexdigest()
1547
+ if key in self._capacities:
1548
+ self._capacities.move_to_end(key)
1549
+ return self._capacities[key]
1550
+ result = self._capacity(request)
1551
+ self._capacities[key] = result
1552
+ if len(self._capacities) > 4:
1553
+ self._capacities.popitem(last=False)
1554
+ return result
1555
+
1556
+ def _capacity(self, request: CapacityRequest):
1557
+ result = {
1558
+ "texts": 0,
1559
+ "available_pairs": 0,
1560
+ "unsupported_texts": 0,
1561
+ "unsupported_examples": [],
1562
+ }
1563
+ if not required_codepoints(request.text):
1564
+ return result
1565
+ with tempfile.TemporaryDirectory(prefix="synthocr-capacity-") as directory:
1566
+ raw, processed = Path(directory) / "raw", Path(directory) / "processed"
1567
+ _custom_source(
1568
+ raw, request.text, _language_tags(request.language, request.languages)[0]
1569
+ )
1570
+ config = replace(
1571
+ self.config,
1572
+ data=replace(
1573
+ self.config.data,
1574
+ raw=str(raw),
1575
+ processed=str(processed),
1576
+ source_languages={},
1577
+ normalization="NFC",
1578
+ deduplicate=True,
1579
+ ),
1580
+ )
1581
+ prepare(config)
1582
+ coverage = self._coverage
1583
+ fonts = self.fonts()
1584
+ with sqlite3.connect(
1585
+ (processed / "texts.sqlite3").as_uri() + "?mode=ro", uri=True
1586
+ ) as db:
1587
+ db.execute("PRAGMA cache_size=-4096")
1588
+ for (text,) in db.execute("SELECT text FROM texts ORDER BY id"):
1589
+ needed = required_codepoints(text)
1590
+ compatible = sum(not coverage.missing(font, needed) for font in fonts)
1591
+ result["texts"] += 1
1592
+ result["available_pairs"] += compatible
1593
+ if compatible == 0:
1594
+ result["unsupported_texts"] += 1
1595
+ if len(result["unsupported_examples"]) < 3:
1596
+ result["unsupported_examples"].append(text[:120])
1597
+ return result
1598
+
1599
+ def preview(self, request: PreviewRequest):
1600
+ # Keep native font allocations bounded even when the browser types quickly.
1601
+ with self.preview_lock:
1602
+ return self._preview(request)
1603
+
1604
+ def _corpus_preview(self, request):
1605
+ """Read one indexed label; never prepare or count a corpus in an HTTP request."""
1606
+ tags = _language_tags(request.language, request.languages)
1607
+ excluded = sorted({*self.config.generation.exclude_genres, "spelling"})
1608
+ path = self.config.path(self.config.data.processed) / "texts.sqlite3"
1609
+ message = "Prepare the corpus with synthocr prepare, or upload a text file"
1610
+ if not path.is_file():
1611
+ raise HTTPException(400, message)
1612
+ db = None
1613
+ try:
1614
+ db = sqlite3.connect(path.as_uri() + "?mode=ro", uri=True)
1615
+ db.execute("PRAGMA cache_size=-1024")
1616
+ db.execute("PRAGMA temp_store=FILE")
1617
+
1618
+ def one(selected, offset, representative=False):
1619
+ conditions, parameters = [], []
1620
+ if selected:
1621
+ conditions.append(f"language IN ({','.join('?' for _ in selected)})")
1622
+ parameters.extend(selected)
1623
+ conditions.append(f"genre NOT IN ({','.join('?' for _ in excluded)})")
1624
+ parameters.extend(excluded)
1625
+ if representative:
1626
+ conditions.append("genre IN ('prose','dialogue') AND length(text)>=12")
1627
+ parameters.append(offset)
1628
+ # Use the language index when available and stop after at most
1629
+ # 32 eligible labels. Avoid sorting corpora prepared without deduplication.
1630
+ return db.execute(
1631
+ "SELECT text,language FROM texts WHERE "
1632
+ + " AND ".join(conditions)
1633
+ + " LIMIT 1 OFFSET ?",
1634
+ parameters,
1635
+ ).fetchone()
1636
+
1637
+ selected = [tags[request.index % len(tags)]] if len(tags) > 1 else tags
1638
+ offset = request.index // len(tags) if len(tags) > 1 else request.index
1639
+ # Prefer readable natural text for the preview. Short labels, numbers
1640
+ # and punctuation remain available to generation without restriction.
1641
+ row = (
1642
+ one(selected, offset, True)
1643
+ or one(selected, 0, True)
1644
+ or one(selected, offset)
1645
+ or one(selected, 0)
1646
+ )
1647
+ if row is None and len(tags) > 1:
1648
+ row = one(tags, 0)
1649
+ if row is None:
1650
+ raise HTTPException(400, "No usable corpus text for this language. " + message)
1651
+ return row[0], canonical_language(row[1])
1652
+ except sqlite3.Error as error:
1653
+ raise HTTPException(400, message) from error
1654
+ finally:
1655
+ if db is not None:
1656
+ db.close()
1657
+
1658
+ def _spelling_preview(self, request):
1659
+ tags = _language_tags(request.language, request.languages)
1660
+ if not tags or any(tag not in {"ckb", "kmr"} for tag in tags):
1661
+ raise HTTPException(422, "Spelling examples support Sorani, Kurmanji or both")
1662
+ tag = tags[request.index % len(tags)]
1663
+ # This is the fixed, bundled spelling catalog, rather than a user corpus.
1664
+ cases = list(spelling_cases(tag))
1665
+ case = cases[(request.index // len(tags)) % len(cases)]
1666
+ return case["text"], case["language"], case["attribution"]["source_id"]
1667
+
1668
+ def _preview(self, request: PreviewRequest):
1669
+ if request.language not in LANGUAGES and not LANGUAGE.fullmatch(request.language):
1670
+ raise HTTPException(422, "Choose a valid language")
1671
+ default = {
1672
+ "sorani": "سڵاو جیهان",
1673
+ "ckb": "سڵاو جیهان",
1674
+ "kurmanji": "Silav dinya",
1675
+ "kmr": "Silav dinya",
1676
+ "both": "سڵاو · Silav",
1677
+ "all": "سڵاو Hello 123",
1678
+ "ar": "مرحبا بالعالم",
1679
+ "fa": "سلام دنیا",
1680
+ "en": "Hello world",
1681
+ "he": "שלום עולם",
1682
+ "hi": "नमस्ते दुनिया",
1683
+ "zh": "你好世界",
1684
+ }
1685
+ tags = _language_tags(request.language, request.languages)
1686
+ source_language = tags[0] if len(tags) == 1 else ""
1687
+ case_id = None
1688
+ if request.source == "corpus":
1689
+ original, source_language = self._corpus_preview(request)
1690
+ elif request.source == "spelling":
1691
+ original, source_language, case_id = self._spelling_preview(request)
1692
+ else:
1693
+ if request.languages is not None and not request.text:
1694
+ source_language = tags[request.index % len(tags)]
1695
+ original = request.text or default.get(
1696
+ source_language, default.get(request.language, "123")
1697
+ )
1698
+ first_line = next((line for line in io.StringIO(original) if required_codepoints(line)), "")
1699
+ text = next(chunks(normalize(first_line), self.config.data.max_chars), "")
1700
+ if not required_codepoints(text):
1701
+ raise HTTPException(422, "Enter visible text or numbers")
1702
+ fonts = self.selected_fonts(request.fonts)
1703
+ if request.fonts:
1704
+ # Show the user's chosen fonts in turn as they ask for more examples.
1705
+ offset = request.index % len(fonts)
1706
+ else:
1707
+ # Start with a readable Kurdish face while still using shared coverage selection.
1708
+ preferred = "NotoNaskhArabic" if source_language in {"ckb", "ar"} else "NotoSans["
1709
+ offset = next((i for i, face in enumerate(fonts) if preferred in face.name), 0)
1710
+ face = self._coverage.choose(fonts, text, offset)
1711
+ if face is None:
1712
+ raise HTTPException(422, "No installed font supports this complete line")
1713
+ settings = replace(
1714
+ self.config.generation,
1715
+ font_size=request.font_size,
1716
+ direction="auto",
1717
+ language="",
1718
+ **{name: getattr(request, name) for name in AUGMENTATION_FIELDS},
1719
+ max_width=2048,
1720
+ max_height=256,
1721
+ max_pixels=524_288,
1722
+ )
1723
+
1724
+ def render(appearance):
1725
+ image, info = Renderer(appearance, cache_size=1).render(
1726
+ text, face, request.seed, source_language
1727
+ )
1728
+ try:
1729
+ with io.BytesIO() as buffer:
1730
+ image.save(buffer, format="PNG")
1731
+ encoded = base64.b64encode(buffer.getvalue()).decode("ascii")
1732
+ finally:
1733
+ image.close()
1734
+ return {"image": f"data:image/png;base64,{encoded}", **info}
1735
+
1736
+ # Hold the sample, font and seed fixed while comparing effects. Only one
1737
+ # decoded image is alive at once, bounded by the preview's pixel limit.
1738
+ before = render(replace(settings, augmentation=0)) if not request.augmentation else None
1739
+ if request.compare:
1740
+ # Hidden recipe selectors retain their existing thumbnails while
1741
+ # disabled. Do not serialize a catalog of identical clean images.
1742
+ presets = list(DOCUMENT_PRESETS) if request.augmentation else []
1743
+ else:
1744
+ presets = request.background_presets
1745
+ variants = []
1746
+ distribution = background_distribution(settings)
1747
+ for preset in presets:
1748
+ rendered = (
1749
+ render(replace(settings, background_presets=[preset], background_weights={}))
1750
+ if request.augmentation
1751
+ else before
1752
+ )
1753
+ if rendered["augmentation"]:
1754
+ rendered["augmentation"]["background_probability"] = distribution.get(preset, 0)
1755
+ variants.append(
1756
+ {
1757
+ "preset": preset,
1758
+ "name": DOCUMENT_PRESETS[preset]["label"],
1759
+ "text": text,
1760
+ "font": face.name,
1761
+ **rendered,
1762
+ }
1763
+ )
1764
+ if variants:
1765
+ selected = next((name for name, share in distribution.items() if share > 0), presets[0])
1766
+ after = next(variant for variant in variants if variant["preset"] == selected)
1767
+ else:
1768
+ after = render(settings) if request.augmentation else before
1769
+ if not request.compare or request.augmentation:
1770
+ variants.append(
1771
+ {
1772
+ "preset": "",
1773
+ "name": "Custom background",
1774
+ "text": text,
1775
+ "font": face.name,
1776
+ **after,
1777
+ }
1778
+ )
1779
+ if before is None:
1780
+ # Bounds can shrink a long rotated line. Compare it with clean ink
1781
+ # at that same effective point size rather than a larger baseline.
1782
+ font_size = min(variant["font_size"] for variant in variants)
1783
+ before = render(replace(settings, augmentation=0, font_size=font_size))
1784
+ return {
1785
+ **after,
1786
+ "text": text,
1787
+ "font": face.name,
1788
+ "wrapped": text != normalize(original),
1789
+ "source": request.source,
1790
+ "source_language": source_language,
1791
+ "case_id": case_id,
1792
+ "seed": request.seed,
1793
+ "before": {"text": text, "font": face.name, **before},
1794
+ "variants": variants,
1795
+ }
1796
+
1797
+ def _sample_offset(self, job, manifest, offset):
1798
+ if job["status"] != "completed" or not offset:
1799
+ return 0, 0
1800
+ info = manifest.stat()
1801
+ fingerprint = hashlib.sha256(
1802
+ f"{info.st_dev}:{info.st_ino}:{info.st_size}:{info.st_mtime_ns}".encode()
1803
+ ).hexdigest()[:16]
1804
+ index = self.cache / f"{job['id']}-{fingerprint}.index"
1805
+ with self.archive_lock:
1806
+ if index.is_symlink():
1807
+ raise HTTPException(400, "Invalid gallery index")
1808
+ if not index.exists():
1809
+ # Eight bytes per 128 records, rather than an in-memory row index.
1810
+ self._trim_cache(reserve=(100_000 // INDEX_STRIDE + 1) * 8)
1811
+ temporary = self.cache / f"{uuid4().hex}.tmp"
1812
+ try:
1813
+ with manifest.open("rb") as stream, temporary.open("xb") as target:
1814
+ line = 0
1815
+ while True:
1816
+ position = stream.tell()
1817
+ if not stream.readline():
1818
+ break
1819
+ if line >= 100_000:
1820
+ raise HTTPException(400, "Dataset exceeds the studio sample limit")
1821
+ if line % INDEX_STRIDE == 0:
1822
+ target.write(struct.pack("<Q", position))
1823
+ line += 1
1824
+ os.replace(temporary, index)
1825
+ finally:
1826
+ temporary.unlink(missing_ok=True)
1827
+ index.touch()
1828
+ entries = index.stat().st_size // 8
1829
+ if not entries:
1830
+ return 0, 0
1831
+ block = min(offset // INDEX_STRIDE, entries - 1)
1832
+ with index.open("rb") as stream:
1833
+ stream.seek(block * 8)
1834
+ return struct.unpack("<Q", stream.read(8))[0], block * INDEX_STRIDE
1835
+
1836
+ def samples(self, job, offset, limit):
1837
+ path = self.dataset(job) / "manifest.jsonl"
1838
+ rows = []
1839
+ if path.is_file() and not path.is_symlink():
1840
+ position, first = self._sample_offset(job, path, offset)
1841
+ with path.open("rb") as stream:
1842
+ stream.seek(position)
1843
+ for index, line in enumerate(stream, first):
1844
+ if index < offset:
1845
+ continue
1846
+ if len(rows) >= limit:
1847
+ break
1848
+ try:
1849
+ row = json.loads(line)
1850
+ except ValueError as error:
1851
+ if job["status"] == "completed":
1852
+ raise HTTPException(400, "Invalid sample in dataset") from error
1853
+ break # A running worker may still be flushing its final line.
1854
+ if not isinstance(row, dict) or not isinstance(row.get("image"), str):
1855
+ raise HTTPException(400, "Invalid sample in dataset")
1856
+ filename = Path(row["image"]).name
1857
+ if not IMAGE.fullmatch(filename):
1858
+ raise HTTPException(400, "Invalid image name in dataset")
1859
+ rows.append(
1860
+ {
1861
+ key: row.get(key)
1862
+ for key in ("id", "text", "font", "language", "split", "augmentation")
1863
+ }
1864
+ | {
1865
+ "image": f"/api/jobs/{job['id']}/images/{filename}",
1866
+ "thumbnail": f"/api/jobs/{job['id']}/thumbnails/{filename}",
1867
+ }
1868
+ )
1869
+ return {"samples": rows, "total": job["generated"], "offset": offset, "limit": limit}
1870
+
1871
+ def image_path(self, job, filename):
1872
+ if not IMAGE.fullmatch(filename):
1873
+ raise HTTPException(404, "Image not found")
1874
+ dataset = self.dataset(job)
1875
+ directory = dataset / "images"
1876
+ path = directory / filename
1877
+ if (
1878
+ directory.is_symlink()
1879
+ or path.is_symlink()
1880
+ or not path.is_file()
1881
+ or path.resolve().parent != directory
1882
+ ):
1883
+ raise HTTPException(404, "Image not found")
1884
+ return path
1885
+
1886
+ def thumbnail(self, job, filename):
1887
+ path = self.image_path(job, filename)
1888
+ with self.preview_lock, Image.open(path) as image:
1889
+ if image.width * image.height > 16_777_216:
1890
+ raise HTTPException(400, "Image exceeds the supported preview size")
1891
+ image.thumbnail((768, 192), Image.Resampling.LANCZOS, reducing_gap=3)
1892
+ with io.BytesIO() as buffer:
1893
+ image.save(buffer, format="PNG")
1894
+ return Response(
1895
+ buffer.getvalue(),
1896
+ media_type="image/png",
1897
+ headers={"Cache-Control": "private, max-age=3600"},
1898
+ )
1899
+
1900
+ def download(self, job, format, *, prepare_only=False):
1901
+ if job["status"] != "completed":
1902
+ raise HTTPException(409, "Downloads are available after generation and verification")
1903
+ dataset = self.dataset(job)
1904
+ with self.archive_lock:
1905
+ verify(dataset)
1906
+ prepared = {
1907
+ "url": f"/api/jobs/{job['id']}/download?format={format}",
1908
+ "filename": f"{job['name']}-{format}.zip"
1909
+ if format != "manifest"
1910
+ else f"{job['name']}-manifest.jsonl",
1911
+ }
1912
+ if format == "manifest":
1913
+ if prepare_only:
1914
+ return prepared
1915
+ return FileResponse(
1916
+ dataset / "manifest.jsonl",
1917
+ media_type="application/x-ndjson",
1918
+ filename=f"{job['name']}-manifest.jsonl",
1919
+ )
1920
+ digests = [
1921
+ sha256(dataset / name) for name in ("manifest.jsonl", "run.json", "stats.json")
1922
+ ]
1923
+ for name in ("source.txt", "source.jsonl", "source.csv", "source-upload.json"):
1924
+ if (dataset / name).is_file():
1925
+ digests.append(sha256(dataset / name))
1926
+ digest = hashlib.sha256("".join(digests).encode()).hexdigest()[:16]
1927
+ archive = self.cache / f"{job['id']}-{digest}-{format}.zip"
1928
+ if archive.is_symlink():
1929
+ raise HTTPException(400, "Invalid download archive")
1930
+ if not archive.exists():
1931
+ # The maintained exporter hardlinks images on this filesystem.
1932
+ # Remove its attributed view after archiving instead of keeping
1933
+ # four permanent copies of each generated dataset.
1934
+ with tempfile.TemporaryDirectory(prefix=".export-", dir=self.metadata) as directory:
1935
+ source = dataset
1936
+ if format != "dataset":
1937
+ retained_sources = tuple(
1938
+ dataset / name
1939
+ for name in (
1940
+ "source.txt",
1941
+ "source.jsonl",
1942
+ "source.csv",
1943
+ "source-upload.json",
1944
+ )
1945
+ if (dataset / name).is_file()
1946
+ )
1947
+ retained_bytes = sum(path.stat().st_size for path in retained_sources)
1948
+ # Retained raw inputs are part of every attributed export.
1949
+ # Account for them before creating or copying its view.
1950
+ self._trim_cache(reserve=retained_bytes)
1951
+ source = Path(directory) / "view"
1952
+ try:
1953
+ export(
1954
+ dataset,
1955
+ source,
1956
+ format,
1957
+ max_bytes=self.cache_limit - retained_bytes,
1958
+ )
1959
+ except ValueError as error:
1960
+ if str(error).startswith("Export exceeds the configured byte limit"):
1961
+ raise HTTPException(
1962
+ 413,
1963
+ "This export exceeds the studio cache limit. Download the "
1964
+ "manifest, use the dataset in the output folder, or restart "
1965
+ "with a larger --cache-mb.",
1966
+ ) from error
1967
+ raise
1968
+ for path in retained_sources:
1969
+ shutil.copyfile(path, source / path.name)
1970
+ self._trim_cache(reserve=_archive_size(source))
1971
+ temporary = self.cache / f"{uuid4().hex}.tmp"
1972
+ try:
1973
+ _stored_zip(source, temporary)
1974
+ os.replace(temporary, archive)
1975
+ finally:
1976
+ temporary.unlink(missing_ok=True)
1977
+ archive.touch()
1978
+ if prepare_only:
1979
+ return prepared
1980
+ self._leased[archive] = self._leased.get(archive, 0) + 1
1981
+ return _ArchiveResponse(
1982
+ archive,
1983
+ media_type="application/zip",
1984
+ filename=f"{job['name']}-{format}.zip",
1985
+ release=self._release_archive,
1986
+ )
1987
+
1988
+
1989
+ def create_app(
1990
+ config: Config,
1991
+ output_root: Path | None = None,
1992
+ *,
1993
+ cache_mb: int = 256,
1994
+ output_locations: dict[str, Path] | None = None,
1995
+ ) -> FastAPI:
1996
+ studio = Studio(config, output_root, cache_mb=cache_mb, output_locations=output_locations)
1997
+
1998
+ @asynccontextmanager
1999
+ async def lifespan(app):
2000
+ try:
2001
+ yield
2002
+ finally:
2003
+ studio.close()
2004
+
2005
+ app = FastAPI(title="SynthOCR Studio", version=__version__, lifespan=lifespan)
2006
+ app.state.studio = studio
2007
+ app.add_middleware(TrustedHostMiddleware, allowed_hosts=["localhost", "127.0.0.1", "[::1]"])
2008
+
2009
+ @app.middleware("http")
2010
+ async def local_requests(request: Request, call_next):
2011
+ if request.method not in {"GET", "HEAD", "OPTIONS"}:
2012
+ origin = request.headers.get("origin")
2013
+ if origin:
2014
+ try:
2015
+ parsed = urlsplit(origin)
2016
+ target = request.url
2017
+ same_origin = (
2018
+ parsed.scheme == target.scheme
2019
+ and parsed.username is None
2020
+ and parsed.password is None
2021
+ and not parsed.query
2022
+ and not parsed.fragment
2023
+ and parsed.path in {"", "/"}
2024
+ and parsed.hostname == target.hostname
2025
+ and (parsed.port or (443 if parsed.scheme == "https" else 80))
2026
+ == (target.port or (443 if target.scheme == "https" else 80))
2027
+ )
2028
+ except ValueError:
2029
+ same_origin = False
2030
+ if not same_origin:
2031
+ return JSONResponse({"detail": "Use the studio's own local browser page"}, 403)
2032
+ if request.headers.get("sec-fetch-site") == "cross-site":
2033
+ return JSONResponse({"detail": "Cross-site requests are not allowed"}, 403)
2034
+ length = request.headers.get("content-length")
2035
+ upload = request.method == "POST" and request.url.path == "/api/uploads"
2036
+ remove_upload = request.method == "DELETE" and re.fullmatch(
2037
+ r"/api/(?:uploads/[a-f0-9]{32}|fonts/[a-f0-9]{64})", request.url.path
2038
+ )
2039
+ if request.method == "POST" and request.url.path == "/api/fonts":
2040
+ if length is not None and (not length.isdigit() or int(length) > FONT_UPLOAD_BYTES):
2041
+ return JSONResponse({"detail": "Font files must be at most 32 MiB"}, 413)
2042
+ elif upload:
2043
+ if length is not None and (not length.isdigit() or int(length) > UPLOAD_BYTES):
2044
+ return JSONResponse({"detail": "Uploaded files must be at most 50 MiB"}, 413)
2045
+ elif remove_upload:
2046
+ if length not in {None, "0"}:
2047
+ return JSONResponse({"detail": "Removal does not accept a body"}, 413)
2048
+ elif length is None or not length.isdigit() or int(length) > 256 * 1024:
2049
+ return JSONResponse({"detail": "Request body must be at most 256 KiB"}, 413)
2050
+ response = await call_next(request)
2051
+ response.headers["X-Content-Type-Options"] = "nosniff"
2052
+ response.headers["Referrer-Policy"] = "same-origin"
2053
+ response.headers.setdefault("Cache-Control", "no-store")
2054
+ return response
2055
+
2056
+ @app.exception_handler(ValueError)
2057
+ async def invalid_operation(request, error):
2058
+ return JSONResponse({"detail": str(error)}, 400)
2059
+
2060
+ @app.exception_handler(OSError)
2061
+ async def inaccessible_output(request, error):
2062
+ return JSONResponse({"detail": f"Cannot access output data: {error}"}, 400)
2063
+
2064
+ @app.get("/api/info")
2065
+ def info():
2066
+ corpora = []
2067
+ languages = [
2068
+ {"id": key, "name": name}
2069
+ for key, name in (
2070
+ ("sorani", "Sorani · سۆرانی"),
2071
+ ("kurmanji", "Kurmanji · Kurmancî"),
2072
+ ("both", "Both Kurdish varieties"),
2073
+ ("all", "All source languages"),
2074
+ )
2075
+ ]
2076
+ registry = config.path(config.data.processed) / "corpora.json"
2077
+ if registry.is_file() and registry.stat().st_size < 1024 * 1024:
2078
+ try:
2079
+ for corpus in json.loads(registry.read_text(encoding="utf-8"))["corpora"]:
2080
+ tag, records = corpus["language"], corpus["records"]
2081
+ if not isinstance(tag, str) or type(records) is not int or records < 1:
2082
+ continue
2083
+ if not tag or LANGUAGE.fullmatch(tag):
2084
+ corpora.append({"language": tag, "records": records})
2085
+ if tag not in {"ckb", "kmr", ""}:
2086
+ languages.append({"id": tag, "name": tag, "records": records})
2087
+ except OSError, ValueError, TypeError, KeyError:
2088
+ pass
2089
+ return {
2090
+ "version": __version__,
2091
+ "output_root": str(studio.output_root),
2092
+ "output_locations": [
2093
+ {
2094
+ "id": name,
2095
+ "label": "Dataset storage"
2096
+ if name == "default"
2097
+ else name.replace("-", " ").capitalize(),
2098
+ }
2099
+ for name in studio.output_locations
2100
+ ],
2101
+ "fonts_count": len(studio.fonts()),
2102
+ "raqm": features.check_feature("raqm"),
2103
+ "languages": languages,
2104
+ "corpora": corpora,
2105
+ "storage": studio.storage(),
2106
+ "upload_limit_bytes": UPLOAD_BYTES,
2107
+ "upload_formats": ["txt", "jsonl", "csv"],
2108
+ "document_presets": DOCUMENT_PRESETS,
2109
+ "document_layers": [
2110
+ {"id": name, "label": label, "description": description}
2111
+ for name, (label, description) in DOCUMENT_LAYER_LABELS.items()
2112
+ ],
2113
+ "image_presets": IMAGE_PRESETS,
2114
+ "recommended_preset": "balanced",
2115
+ "defaults": JobRequest().model_dump(exclude={"name", "source", "text", "upload_id"}),
2116
+ }
2117
+
2118
+ @app.get("/api/directories")
2119
+ def directories(
2120
+ location: str = Query("default", max_length=32),
2121
+ path: str = Query("", max_length=512),
2122
+ ):
2123
+ return studio.directories(location, path)
2124
+
2125
+ @app.post("/api/directories", status_code=201)
2126
+ def create_directory(request: DirectoryRequest):
2127
+ return studio.create_directory(request)
2128
+
2129
+ @app.get("/api/fonts")
2130
+ def list_fonts():
2131
+ return {"fonts": studio.font_catalog()[0]}
2132
+
2133
+ @app.post("/api/fonts", status_code=201)
2134
+ async def add_font(request: Request, filename: str = Query(..., min_length=1, max_length=255)):
2135
+ return await studio.add_font(request, filename)
2136
+
2137
+ @app.delete("/api/fonts/{digest}")
2138
+ def remove_font(digest: str = FastAPIPath(pattern=r"^[a-f0-9]{64}$")):
2139
+ return studio.remove_font(digest)
2140
+
2141
+ @app.post("/api/uploads", status_code=201)
2142
+ async def upload(
2143
+ request: Request,
2144
+ filename: str = Query(..., min_length=1, max_length=255),
2145
+ language: str = "sorani",
2146
+ languages: Annotated[
2147
+ list[Literal["ckb", "kmr", "ar", "en"]] | None, Query(min_length=1, max_length=4)
2148
+ ] = None,
2149
+ ):
2150
+ return await studio.upload(request, filename, language, languages)
2151
+
2152
+ @app.delete("/api/uploads/{identifier}")
2153
+ def remove_upload(identifier: str):
2154
+ return studio.remove_upload(identifier)
2155
+
2156
+ @app.post("/api/preview")
2157
+ def preview(request: PreviewRequest):
2158
+ return studio.preview(request)
2159
+
2160
+ @app.post("/api/capacity")
2161
+ def capacity(request: CapacityRequest):
2162
+ return studio.capacity(request)
2163
+
2164
+ @app.post("/api/jobs", status_code=202)
2165
+ def start(request: JobRequest):
2166
+ return studio.start(request)
2167
+
2168
+ @app.get("/api/jobs")
2169
+ def jobs():
2170
+ return {"jobs": studio.jobs()}
2171
+
2172
+ @app.get("/api/jobs/{identifier}")
2173
+ def job(identifier: str):
2174
+ return studio.job(identifier)
2175
+
2176
+ @app.post("/api/jobs/{identifier}/cancel")
2177
+ def cancel(identifier: str):
2178
+ return studio.cancel(identifier)
2179
+
2180
+ @app.get("/api/jobs/{identifier}/samples")
2181
+ def samples(identifier: str, offset: int = Query(0, ge=0), limit: int = Query(12, ge=1, le=24)):
2182
+ return studio.samples(studio.job(identifier), offset, limit)
2183
+
2184
+ @app.get("/api/jobs/{identifier}/images/{filename}")
2185
+ def image(identifier: str, filename: str):
2186
+ path = studio.image_path(studio.job(identifier), filename)
2187
+ if path.suffix == ".tif":
2188
+ # Browsers cannot display TIFF consistently; keep the original file in downloads.
2189
+ with Image.open(path) as source:
2190
+ if source.width * source.height > 16_777_216:
2191
+ raise HTTPException(400, "Image exceeds the supported preview size")
2192
+ with io.BytesIO() as buffer:
2193
+ source.save(buffer, format="PNG")
2194
+ return Response(buffer.getvalue(), media_type="image/png")
2195
+ return FileResponse(path)
2196
+
2197
+ @app.get("/api/jobs/{identifier}/thumbnails/{filename}")
2198
+ def thumbnail(identifier: str, filename: str):
2199
+ return studio.thumbnail(studio.job(identifier), filename)
2200
+
2201
+ @app.post("/api/jobs/{identifier}/download")
2202
+ def prepare_download(
2203
+ identifier: str,
2204
+ format: DOWNLOAD_FORMAT = "dataset",
2205
+ ):
2206
+ # Check and prepare before triggering the native browser download. Errors
2207
+ # remain in the form instead of replacing it with a JSON error page.
2208
+ try:
2209
+ return studio.download(studio.job(identifier), format, prepare_only=True)
2210
+ except HTTPException as error:
2211
+ if error.status_code == 413:
2212
+ raise HTTPException(
2213
+ 413,
2214
+ f"This ZIP exceeds the {studio.cache_limit // 1048576} MiB download limit. "
2215
+ "Try fewer images or JPEG format in a new run. "
2216
+ "You can still download the JSONL labels.",
2217
+ ) from error
2218
+ if error.status_code == 503:
2219
+ raise HTTPException(
2220
+ 503, "Other downloads are in progress. Try again after they finish."
2221
+ ) from error
2222
+ raise
2223
+
2224
+ @app.get("/api/jobs/{identifier}/download")
2225
+ def download(
2226
+ identifier: str,
2227
+ format: DOWNLOAD_FORMAT = "dataset",
2228
+ ):
2229
+ return studio.download(studio.job(identifier), format)
2230
+
2231
+ def ui_font(name: str):
2232
+ fonts = {
2233
+ "ui-latin.ttf": "fonts/open/notosans/NotoSans[wdth,wght].ttf",
2234
+ "ui-kurdish.ttf": "fonts/open/notonaskharabic/NotoNaskhArabic[wght].ttf",
2235
+ }
2236
+ if name not in fonts:
2237
+ raise HTTPException(404, "Font not found")
2238
+ path = config.path(fonts[name])
2239
+ if not path.is_file():
2240
+ raise HTTPException(404, "Font not installed")
2241
+ return FileResponse(path, media_type="font/ttf")
2242
+
2243
+ @app.get("/assets/ui-latin.ttf")
2244
+ def latin_font():
2245
+ return ui_font("ui-latin.ttf")
2246
+
2247
+ @app.get("/assets/ui-kurdish.ttf")
2248
+ def kurdish_font():
2249
+ return ui_font("ui-kurdish.ttf")
2250
+
2251
+ static = Path(str(files("synthocr").joinpath("static")))
2252
+ if static.is_dir():
2253
+ app.mount("/assets", StaticFiles(directory=static), name="assets")
2254
+
2255
+ @app.get("/")
2256
+ def index():
2257
+ return FileResponse(static / "index.html")
2258
+
2259
+ return app