synthocr 2.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- synthocr/__init__.py +3 -0
- synthocr/__main__.py +3 -0
- synthocr/cli.py +540 -0
- synthocr/config.py +589 -0
- synthocr/corpora.py +160 -0
- synthocr/corpus_store.py +222 -0
- synthocr/downloads.py +77 -0
- synthocr/experiments.py +399 -0
- synthocr/font_catalog.json +579 -0
- synthocr/font_profiles.json +18 -0
- synthocr/fonts.py +225 -0
- synthocr/home.py +150 -0
- synthocr/kurdish_characters.json +442 -0
- synthocr/kurdish_spelling.json +45 -0
- synthocr/lexicons.json +82 -0
- synthocr/pipeline.py +512 -0
- synthocr/preview.py +112 -0
- synthocr/render.py +555 -0
- synthocr/spelling.py +111 -0
- synthocr/static/app.css +1111 -0
- synthocr/static/app.js +1759 -0
- synthocr/static/brand.svg +5 -0
- synthocr/static/index.html +773 -0
- synthocr/static/theme.js +28 -0
- synthocr/text.py +437 -0
- synthocr/verify.py +307 -0
- synthocr/web.py +2259 -0
- synthocr-2.1.0.dist-info/METADATA +261 -0
- synthocr-2.1.0.dist-info/RECORD +34 -0
- synthocr-2.1.0.dist-info/WHEEL +4 -0
- synthocr-2.1.0.dist-info/entry_points.txt +3 -0
- synthocr-2.1.0.dist-info/licenses/LICENSE-APACHE +202 -0
- synthocr-2.1.0.dist-info/licenses/LICENSE-MIT +21 -0
- synthocr-2.1.0.dist-info/licenses/NOTICE +25 -0
synthocr/web.py
ADDED
|
@@ -0,0 +1,2259 @@
|
|
|
1
|
+
"""A local FastAPI studio over the maintained, bounded SynthOCR pipeline."""
|
|
2
|
+
|
|
3
|
+
import base64
|
|
4
|
+
import hashlib
|
|
5
|
+
import heapq
|
|
6
|
+
import io
|
|
7
|
+
import json
|
|
8
|
+
import ntpath
|
|
9
|
+
import os
|
|
10
|
+
import re
|
|
11
|
+
import shutil
|
|
12
|
+
import sqlite3
|
|
13
|
+
import struct
|
|
14
|
+
import tempfile
|
|
15
|
+
import threading
|
|
16
|
+
import time
|
|
17
|
+
import zlib
|
|
18
|
+
from collections import OrderedDict
|
|
19
|
+
from concurrent.futures import ThreadPoolExecutor
|
|
20
|
+
from contextlib import asynccontextmanager
|
|
21
|
+
from dataclasses import replace
|
|
22
|
+
from datetime import UTC, datetime
|
|
23
|
+
from importlib.resources import files
|
|
24
|
+
from math import isfinite
|
|
25
|
+
from pathlib import Path
|
|
26
|
+
from typing import Annotated, Literal
|
|
27
|
+
from urllib.parse import urlsplit
|
|
28
|
+
from uuid import uuid4
|
|
29
|
+
|
|
30
|
+
from anyio import CancelScope
|
|
31
|
+
from fastapi import FastAPI, HTTPException, Query, Request
|
|
32
|
+
from fastapi import Path as FastAPIPath
|
|
33
|
+
from fastapi.responses import FileResponse, JSONResponse, Response
|
|
34
|
+
from fastapi.staticfiles import StaticFiles
|
|
35
|
+
from fontTools.ttLib import TTCollection, TTFont, TTLibError
|
|
36
|
+
from PIL import Image, ImageFont, features
|
|
37
|
+
from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
|
|
38
|
+
from starlette.concurrency import run_in_threadpool
|
|
39
|
+
from starlette.middleware.trustedhost import TrustedHostMiddleware
|
|
40
|
+
|
|
41
|
+
from . import __version__
|
|
42
|
+
from .config import (
|
|
43
|
+
AUGMENTATION_FIELDS,
|
|
44
|
+
DOCUMENT_LAYER_LABELS,
|
|
45
|
+
DOCUMENT_PRESETS,
|
|
46
|
+
IMAGE_PRESETS,
|
|
47
|
+
Config,
|
|
48
|
+
background_distribution,
|
|
49
|
+
validate_augmentation,
|
|
50
|
+
)
|
|
51
|
+
from .corpus_store import canonical_language, resolve_source_language
|
|
52
|
+
from .downloads import sha256
|
|
53
|
+
from .fonts import FONT_SUFFIXES, Coverage, discover, font_ids, select
|
|
54
|
+
from .pipeline import GenerationCancelled, export, generate
|
|
55
|
+
from .render import Renderer
|
|
56
|
+
from .spelling import spelling_cases
|
|
57
|
+
from .text import chunks, normalize, prepare, records, required_codepoints
|
|
58
|
+
from .verify import dataset_files, verify
|
|
59
|
+
|
|
60
|
+
NAME = re.compile(r"[A-Za-z0-9][A-Za-z0-9_-]{0,63}\Z")
|
|
61
|
+
LOCATION = re.compile(r"[a-z][a-z0-9-]{0,31}\Z")
|
|
62
|
+
FOLDER_NAME = re.compile(r'[^\x00-\x1f\x7f<>:"/\\|?*]{1,80}\Z')
|
|
63
|
+
DIRECTORY_LIMIT = 200
|
|
64
|
+
DIRECTORY_SCAN_LIMIT = 2000
|
|
65
|
+
JOB_ID = re.compile(r"[a-f0-9]{32}\Z")
|
|
66
|
+
DOWNLOAD_FORMAT = Literal["dataset", "csv", "imagefolder", "tesseract", "manifest"]
|
|
67
|
+
LANGUAGE = re.compile(r"[A-Za-z]{2,8}(?:-[A-Za-z0-9]{1,8})*\Z")
|
|
68
|
+
IMAGE = re.compile(r"[0-9]{8}\.(?:png|jpg|webp|tif)\Z")
|
|
69
|
+
CACHE_FILES = 128
|
|
70
|
+
INDEX_STRIDE = 128
|
|
71
|
+
UPLOAD_BYTES = 50 * 1024 * 1024
|
|
72
|
+
FONT_UPLOAD_BYTES = 32 * 1024 * 1024
|
|
73
|
+
USER_FONT_LIMIT = 200
|
|
74
|
+
FONT_ID = re.compile(r"[a-f0-9]{64}:[0-9]{1,4}\Z")
|
|
75
|
+
# A short sample of each page language; a font that draws it all is marked as
|
|
76
|
+
# supporting that language. Generation still checks every label separately.
|
|
77
|
+
LANGUAGE_SAMPLES = {
|
|
78
|
+
"ckb": "ئابپتجچحخدرڕزژسشعغفڤقکگلڵمنوۆهەیێ",
|
|
79
|
+
"kmr": "abcçdeêfghiîjklmnopqrsştuûvwxyzABCÇDEÊFGHIÎJKLMNOPQRSŞTUÛVWXYZ",
|
|
80
|
+
"ar": "ابتثجحخدذرزسشصضطظعغفقكلمنهوي",
|
|
81
|
+
"en": "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ",
|
|
82
|
+
}
|
|
83
|
+
UPLOAD_TOTAL_BYTES = 100 * 1024 * 1024
|
|
84
|
+
UPLOAD_FILES = 8
|
|
85
|
+
UPLOAD_TTL = 24 * 60 * 60
|
|
86
|
+
UPLOAD_CHUNK = 64 * 1024
|
|
87
|
+
ACTIVE = {"queued", "preparing", "generating", "verifying", "cancelling"}
|
|
88
|
+
LANGUAGES = {"sorani": ["ckb"], "kurmanji": ["kmr"], "both": ["ckb", "kmr"], "all": []}
|
|
89
|
+
FOCUSED_LANGUAGES = ("ckb", "kmr", "ar", "en")
|
|
90
|
+
LanguageSelection = Annotated[
|
|
91
|
+
list[Literal["ckb", "kmr", "ar", "en"]], Field(min_length=1, max_length=4)
|
|
92
|
+
]
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def _font_display_name(font) -> str:
|
|
96
|
+
try:
|
|
97
|
+
with TTFont(font.path, fontNumber=font.index, lazy=True) as face:
|
|
98
|
+
name = face["name"].getDebugName(4) or face["name"].getDebugName(1)
|
|
99
|
+
except OSError, ValueError, KeyError, TTLibError:
|
|
100
|
+
name = None
|
|
101
|
+
return (name or Path(font.path).stem)[:80]
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def _folder_name(name: str) -> bool:
|
|
105
|
+
if not isinstance(name, str):
|
|
106
|
+
return False
|
|
107
|
+
try:
|
|
108
|
+
if len(name.encode("utf-8")) > 255:
|
|
109
|
+
return False
|
|
110
|
+
except UnicodeEncodeError:
|
|
111
|
+
return False
|
|
112
|
+
return bool(
|
|
113
|
+
FOLDER_NAME.fullmatch(name)
|
|
114
|
+
and not name.startswith(".")
|
|
115
|
+
and name[-1] not in " ."
|
|
116
|
+
and not ntpath.isreserved(name)
|
|
117
|
+
)
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
def _folder_parts(path: str) -> list[str]:
|
|
121
|
+
if not isinstance(path, str) or len(path) > 512:
|
|
122
|
+
raise ValueError("Output folder must be a relative path of at most 512 characters")
|
|
123
|
+
parts = path.split("/") if path else []
|
|
124
|
+
if len(parts) > 8 or any(not _folder_name(part) for part in parts):
|
|
125
|
+
raise ValueError("Choose an output folder within the selected storage location")
|
|
126
|
+
return parts
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
class DirectoryRequest(BaseModel):
|
|
130
|
+
model_config = ConfigDict(extra="forbid")
|
|
131
|
+
|
|
132
|
+
location: str = Field(default="default", pattern=r"^[a-z][a-z0-9-]{0,31}$")
|
|
133
|
+
path: str = Field(default="", max_length=512)
|
|
134
|
+
name: str = Field(min_length=1, max_length=80)
|
|
135
|
+
|
|
136
|
+
@model_validator(mode="after")
|
|
137
|
+
def valid_directory(self):
|
|
138
|
+
_folder_parts(self.path)
|
|
139
|
+
if not _folder_name(self.name):
|
|
140
|
+
raise ValueError("Use a portable folder name without path separators or reserved names")
|
|
141
|
+
_folder_parts("/".join(filter(None, (self.path, self.name))))
|
|
142
|
+
return self
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
class AppearanceRequest(BaseModel):
|
|
146
|
+
model_config = ConfigDict(extra="forbid")
|
|
147
|
+
|
|
148
|
+
augmentation: float = Field(default=0.35, ge=0, le=1, allow_inf_nan=False, strict=True)
|
|
149
|
+
background: Literal["paper", "plain", "gradient", "noise"] = "paper"
|
|
150
|
+
color_mode: Literal["grayscale", "rgb"] = "grayscale"
|
|
151
|
+
background_color: str = "#f5f1e8"
|
|
152
|
+
background_color_end: str = "#e4e9ed"
|
|
153
|
+
text_color: str = "#1a1a1a"
|
|
154
|
+
texture_strength: float = Field(default=0.04, ge=0, le=0.25, allow_inf_nan=False, strict=True)
|
|
155
|
+
rotation: float = Field(default=2, ge=0, le=10, allow_inf_nan=False, strict=True)
|
|
156
|
+
blur: float = Field(default=0.55, ge=0, le=2, allow_inf_nan=False, strict=True)
|
|
157
|
+
contrast: float = Field(default=0.1, ge=0, le=0.5, allow_inf_nan=False, strict=True)
|
|
158
|
+
background_presets: list[str] = Field(default_factory=list, max_length=len(DOCUMENT_PRESETS))
|
|
159
|
+
background_weights: dict[
|
|
160
|
+
str, Annotated[float, Field(ge=0, le=1_000_000, allow_inf_nan=False, strict=True)]
|
|
161
|
+
] = Field(default_factory=dict, max_length=len(DOCUMENT_PRESETS))
|
|
162
|
+
preset_colors: bool = Field(default=True, strict=True)
|
|
163
|
+
paper_texture: float | None = Field(default=None, ge=0, le=1, allow_inf_nan=False, strict=True)
|
|
164
|
+
grain: float | None = Field(default=None, ge=0, le=1, allow_inf_nan=False, strict=True)
|
|
165
|
+
shadow: float | None = Field(default=None, ge=0, le=1, allow_inf_nan=False, strict=True)
|
|
166
|
+
stains: float | None = Field(default=None, ge=0, le=1, allow_inf_nan=False, strict=True)
|
|
167
|
+
creases: float | None = Field(default=None, ge=0, le=1, allow_inf_nan=False, strict=True)
|
|
168
|
+
ruled_lines: float | None = Field(default=None, ge=0, le=1, allow_inf_nan=False, strict=True)
|
|
169
|
+
scan_banding: float | None = Field(default=None, ge=0, le=1, allow_inf_nan=False, strict=True)
|
|
170
|
+
speckles: float | None = Field(default=None, ge=0, le=1, allow_inf_nan=False, strict=True)
|
|
171
|
+
binding_shadow: float | None = Field(default=None, ge=0, le=1, allow_inf_nan=False, strict=True)
|
|
172
|
+
|
|
173
|
+
@model_validator(mode="after")
|
|
174
|
+
def valid_appearance(self):
|
|
175
|
+
validate_augmentation(self)
|
|
176
|
+
return self
|
|
177
|
+
|
|
178
|
+
|
|
179
|
+
class JobRequest(AppearanceRequest):
|
|
180
|
+
name: str | None = Field(default=None, min_length=1, max_length=64)
|
|
181
|
+
output_location: str = Field(default="default", pattern=r"^[a-z][a-z0-9-]{0,31}$")
|
|
182
|
+
output_folder: str = Field(default="", max_length=512)
|
|
183
|
+
language: str = "sorani"
|
|
184
|
+
languages: LanguageSelection | None = None
|
|
185
|
+
source: Literal["corpus", "custom", "spelling", "upload"] = "corpus"
|
|
186
|
+
language_weights: dict[str, float] = Field(default_factory=dict, max_length=8)
|
|
187
|
+
upload_id: str | None = Field(default=None, min_length=32, max_length=32)
|
|
188
|
+
text: str = Field(default="", max_length=200_000)
|
|
189
|
+
samples: int = Field(default=1000, ge=1, le=100_000, strict=True)
|
|
190
|
+
font_size: int = Field(default=48, ge=12, le=128, strict=True)
|
|
191
|
+
fonts: list[str] = Field(default_factory=list, max_length=1000)
|
|
192
|
+
|
|
193
|
+
@field_validator("fonts")
|
|
194
|
+
@classmethod
|
|
195
|
+
def chosen_font_ids(cls, value):
|
|
196
|
+
if any(not FONT_ID.fullmatch(item) for item in value):
|
|
197
|
+
raise ValueError("Fonts are chosen by their ID from /api/fonts")
|
|
198
|
+
return value
|
|
199
|
+
|
|
200
|
+
format: Literal["png", "jpg", "webp", "tif"] = "png"
|
|
201
|
+
seed: int = Field(default=42, ge=-(2**63), le=2**63 - 1, strict=True)
|
|
202
|
+
workers: int = Field(default=1, ge=1, le=4, strict=True)
|
|
203
|
+
train_ratio: float = Field(default=0.9, ge=0, le=1, allow_inf_nan=False)
|
|
204
|
+
validation_ratio: float = Field(default=0.05, ge=0, le=1, allow_inf_nan=False)
|
|
205
|
+
|
|
206
|
+
@model_validator(mode="after")
|
|
207
|
+
def coherent(self):
|
|
208
|
+
if self.name is not None and (not NAME.fullmatch(self.name) or not _folder_name(self.name)):
|
|
209
|
+
raise ValueError("Dataset name may contain letters, numbers, underscores and hyphens")
|
|
210
|
+
_folder_parts(self.output_folder)
|
|
211
|
+
if self.language not in LANGUAGES and not LANGUAGE.fullmatch(self.language):
|
|
212
|
+
raise ValueError("Choose Sorani, Kurmanji, both, all or a valid source language tag")
|
|
213
|
+
tags = _language_tags(self.language, self.languages)
|
|
214
|
+
if self.language_weights and (
|
|
215
|
+
not set(self.language_weights) <= set(tags)
|
|
216
|
+
or any(
|
|
217
|
+
not isfinite(value) or not 0 <= value <= 1_000_000
|
|
218
|
+
for value in self.language_weights.values()
|
|
219
|
+
)
|
|
220
|
+
or not any(self.language_weights.get(tag, 1) > 0 for tag in tags)
|
|
221
|
+
):
|
|
222
|
+
raise ValueError(
|
|
223
|
+
"Language shares must be numbers for the selected languages, "
|
|
224
|
+
"with at least one above zero"
|
|
225
|
+
)
|
|
226
|
+
if self.train_ratio + self.validation_ratio > 1:
|
|
227
|
+
raise ValueError("Training and validation proportions must add up to at most 1")
|
|
228
|
+
if len(self.text.encode("utf-8")) > 200 * 1024:
|
|
229
|
+
raise ValueError("Custom text must be at most 200 KiB")
|
|
230
|
+
if self.source == "custom":
|
|
231
|
+
if len(tags) != 1:
|
|
232
|
+
raise ValueError("Choose one explicit source language for your own text")
|
|
233
|
+
if not required_codepoints(self.text):
|
|
234
|
+
raise ValueError("Enter at least one line containing visible text or numbers")
|
|
235
|
+
elif self.text:
|
|
236
|
+
raise ValueError("Custom text is only accepted with the custom text source")
|
|
237
|
+
if self.source == "spelling" and (
|
|
238
|
+
not tags or any(tag not in {"ckb", "kmr"} for tag in tags)
|
|
239
|
+
):
|
|
240
|
+
raise ValueError("Spelling checks support Sorani, Kurmanji or both")
|
|
241
|
+
if self.source == "upload":
|
|
242
|
+
if self.upload_id is None or not JOB_ID.fullmatch(self.upload_id):
|
|
243
|
+
raise ValueError("Upload a text file before generating")
|
|
244
|
+
elif self.upload_id is not None:
|
|
245
|
+
raise ValueError("Uploaded files are only accepted with the upload source")
|
|
246
|
+
return self
|
|
247
|
+
|
|
248
|
+
|
|
249
|
+
class PreviewRequest(AppearanceRequest):
|
|
250
|
+
model_config = ConfigDict(extra="forbid")
|
|
251
|
+
|
|
252
|
+
text: str = Field(default="", max_length=200_000)
|
|
253
|
+
language: str = "sorani"
|
|
254
|
+
languages: LanguageSelection | None = None
|
|
255
|
+
source: Literal["demo", "corpus", "spelling"] = "demo"
|
|
256
|
+
index: int = Field(default=0, ge=0, le=31, strict=True)
|
|
257
|
+
font_size: int = Field(default=48, ge=12, le=128, strict=True)
|
|
258
|
+
augmentation: float = Field(default=0, ge=0, le=1, allow_inf_nan=False, strict=True)
|
|
259
|
+
seed: int = Field(default=42, ge=-(2**63), le=2**63 - 1, strict=True)
|
|
260
|
+
compare: bool = Field(default=False, strict=True)
|
|
261
|
+
fonts: list[str] = Field(default_factory=list, max_length=1000)
|
|
262
|
+
|
|
263
|
+
@field_validator("fonts")
|
|
264
|
+
@classmethod
|
|
265
|
+
def chosen_font_ids(cls, value):
|
|
266
|
+
if any(not FONT_ID.fullmatch(item) for item in value):
|
|
267
|
+
raise ValueError("Fonts are chosen by their ID from /api/fonts")
|
|
268
|
+
return value
|
|
269
|
+
|
|
270
|
+
@model_validator(mode="after")
|
|
271
|
+
def bounded_text(self):
|
|
272
|
+
if len(self.text.encode("utf-8")) > 200 * 1024:
|
|
273
|
+
raise ValueError("Preview text must be at most 200 KiB")
|
|
274
|
+
tags = _language_tags(self.language, self.languages)
|
|
275
|
+
if self.languages is not None and self.source == "demo" and self.text and len(tags) != 1:
|
|
276
|
+
raise ValueError("Choose one explicit source language for your preview text")
|
|
277
|
+
if self.source == "spelling" and (
|
|
278
|
+
not tags or any(tag not in {"ckb", "kmr"} for tag in tags)
|
|
279
|
+
):
|
|
280
|
+
raise ValueError("Spelling examples support Sorani, Kurmanji or both")
|
|
281
|
+
return self
|
|
282
|
+
|
|
283
|
+
|
|
284
|
+
class CapacityRequest(BaseModel):
|
|
285
|
+
model_config = ConfigDict(extra="forbid")
|
|
286
|
+
|
|
287
|
+
text: str = Field(default="", max_length=200_000)
|
|
288
|
+
language: str = "sorani"
|
|
289
|
+
languages: LanguageSelection | None = None
|
|
290
|
+
|
|
291
|
+
@model_validator(mode="after")
|
|
292
|
+
def coherent(self):
|
|
293
|
+
if len(self.text.encode("utf-8")) > 200 * 1024:
|
|
294
|
+
raise ValueError("Custom text must be at most 200 KiB")
|
|
295
|
+
if len(_language_tags(self.language, self.languages)) != 1:
|
|
296
|
+
raise ValueError("Choose one explicit source language for your own text")
|
|
297
|
+
return self
|
|
298
|
+
|
|
299
|
+
|
|
300
|
+
def _now():
|
|
301
|
+
return datetime.now(UTC).isoformat()
|
|
302
|
+
|
|
303
|
+
|
|
304
|
+
def _atomic_json(path: Path, value: dict):
|
|
305
|
+
temporary = None
|
|
306
|
+
try:
|
|
307
|
+
with tempfile.NamedTemporaryFile(
|
|
308
|
+
"w", dir=path.parent, encoding="utf-8", delete=False
|
|
309
|
+
) as file:
|
|
310
|
+
temporary = Path(file.name)
|
|
311
|
+
json.dump(value, file, ensure_ascii=False)
|
|
312
|
+
file.write("\n")
|
|
313
|
+
# Close before renaming: Windows cannot rename a file that is still open.
|
|
314
|
+
os.replace(temporary, path)
|
|
315
|
+
finally:
|
|
316
|
+
if temporary is not None:
|
|
317
|
+
temporary.unlink(missing_ok=True)
|
|
318
|
+
|
|
319
|
+
|
|
320
|
+
def _language_tags(language, languages=None):
|
|
321
|
+
if languages is not None:
|
|
322
|
+
if (
|
|
323
|
+
not isinstance(languages, list)
|
|
324
|
+
or not 1 <= len(languages) <= 4
|
|
325
|
+
or any(tag not in FOCUSED_LANGUAGES for tag in languages)
|
|
326
|
+
or len(set(languages)) != len(languages)
|
|
327
|
+
):
|
|
328
|
+
raise ValueError(
|
|
329
|
+
"Choose distinct source languages from Sorani, Kurmanji, Arabic and English"
|
|
330
|
+
)
|
|
331
|
+
return list(languages)
|
|
332
|
+
if not isinstance(language, str) or (
|
|
333
|
+
language not in LANGUAGES and not LANGUAGE.fullmatch(language)
|
|
334
|
+
):
|
|
335
|
+
raise ValueError("Choose a valid source language")
|
|
336
|
+
return [canonical_language(tag) for tag in LANGUAGES.get(language, [language])]
|
|
337
|
+
|
|
338
|
+
|
|
339
|
+
def _custom_source(raw: Path, text: str, language: str) -> Path:
|
|
340
|
+
raw.mkdir()
|
|
341
|
+
path = raw / "custom.jsonl"
|
|
342
|
+
with path.open("x", encoding="utf-8") as stream:
|
|
343
|
+
for line, value in enumerate(io.StringIO(text), 1):
|
|
344
|
+
value = value.removesuffix("\n").removesuffix("\r")
|
|
345
|
+
if not value.strip():
|
|
346
|
+
continue
|
|
347
|
+
stream.write(
|
|
348
|
+
json.dumps(
|
|
349
|
+
{
|
|
350
|
+
"text": value,
|
|
351
|
+
"language": language,
|
|
352
|
+
"genre": "user-text",
|
|
353
|
+
"attribution": {
|
|
354
|
+
"source": "User supplied text",
|
|
355
|
+
"line": line,
|
|
356
|
+
"modified": False,
|
|
357
|
+
},
|
|
358
|
+
},
|
|
359
|
+
ensure_ascii=False,
|
|
360
|
+
)
|
|
361
|
+
+ "\n"
|
|
362
|
+
)
|
|
363
|
+
return path
|
|
364
|
+
|
|
365
|
+
|
|
366
|
+
def _lock_output(metadata: Path):
|
|
367
|
+
"""Only one studio may recover and write job history for this output root."""
|
|
368
|
+
path = metadata / ".server.lock"
|
|
369
|
+
if path.is_symlink():
|
|
370
|
+
raise ValueError("Studio lock must not be a symbolic link")
|
|
371
|
+
stream = path.open("a+b")
|
|
372
|
+
try:
|
|
373
|
+
if os.name == "nt":
|
|
374
|
+
import msvcrt
|
|
375
|
+
|
|
376
|
+
stream.seek(0)
|
|
377
|
+
if not stream.read(1):
|
|
378
|
+
stream.write(b"0")
|
|
379
|
+
stream.flush()
|
|
380
|
+
stream.seek(0)
|
|
381
|
+
msvcrt.locking(stream.fileno(), msvcrt.LK_NBLCK, 1)
|
|
382
|
+
else:
|
|
383
|
+
import fcntl
|
|
384
|
+
|
|
385
|
+
fcntl.flock(stream.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
386
|
+
except OSError as error:
|
|
387
|
+
stream.close()
|
|
388
|
+
raise ValueError(
|
|
389
|
+
"Another SynthOCR studio is using this output folder. "
|
|
390
|
+
"Use its browser tab, stop that server, or choose another --output-root."
|
|
391
|
+
) from error
|
|
392
|
+
return stream
|
|
393
|
+
|
|
394
|
+
|
|
395
|
+
def _archive_size(source: Path):
|
|
396
|
+
"""Size of a stored ZIP64, without a dataset-sized entry list."""
|
|
397
|
+
size = 98 # ZIP64 end record, locator and classic end record.
|
|
398
|
+
for path in dataset_files(source):
|
|
399
|
+
if path.is_symlink():
|
|
400
|
+
raise ValueError("Dataset archives cannot contain symbolic links")
|
|
401
|
+
if path.is_file():
|
|
402
|
+
size += (
|
|
403
|
+
path.stat().st_size + 124 + 2 * len(path.relative_to(source).as_posix().encode())
|
|
404
|
+
)
|
|
405
|
+
return size
|
|
406
|
+
|
|
407
|
+
|
|
408
|
+
def _stored_zip(source: Path, destination: Path):
|
|
409
|
+
"""Write ZIP64 without ZipFile's in-memory list of every image entry.
|
|
410
|
+
|
|
411
|
+
PNG/JPEG/WebP/TIFF images are already compressed. Stored entries also avoid
|
|
412
|
+
spending CPU recompressing them; the central directory is spooled to disk.
|
|
413
|
+
"""
|
|
414
|
+
with destination.open("xb") as output, tempfile.TemporaryFile() as directory:
|
|
415
|
+
count = 0
|
|
416
|
+
for path in dataset_files(source):
|
|
417
|
+
if path.is_symlink():
|
|
418
|
+
raise ValueError("Dataset archives cannot contain symbolic links")
|
|
419
|
+
if not path.is_file():
|
|
420
|
+
continue
|
|
421
|
+
name = path.relative_to(source).as_posix().encode("utf-8")
|
|
422
|
+
size, crc = 0, 0
|
|
423
|
+
with path.open("rb") as stream:
|
|
424
|
+
while chunk := stream.read(1024 * 1024):
|
|
425
|
+
size += len(chunk)
|
|
426
|
+
crc = zlib.crc32(chunk, crc)
|
|
427
|
+
offset = output.tell()
|
|
428
|
+
extra = struct.pack("<HHQQ", 1, 16, size, size)
|
|
429
|
+
output.write(
|
|
430
|
+
struct.pack(
|
|
431
|
+
"<IHHHHHIIIHH",
|
|
432
|
+
0x04034B50,
|
|
433
|
+
45,
|
|
434
|
+
0x800,
|
|
435
|
+
0,
|
|
436
|
+
0,
|
|
437
|
+
33,
|
|
438
|
+
crc,
|
|
439
|
+
0xFFFFFFFF,
|
|
440
|
+
0xFFFFFFFF,
|
|
441
|
+
len(name),
|
|
442
|
+
len(extra),
|
|
443
|
+
)
|
|
444
|
+
+ name
|
|
445
|
+
+ extra
|
|
446
|
+
)
|
|
447
|
+
with path.open("rb") as stream:
|
|
448
|
+
shutil.copyfileobj(stream, output, 1024 * 1024)
|
|
449
|
+
extra = struct.pack("<HHQQQ", 1, 24, size, size, offset)
|
|
450
|
+
directory.write(
|
|
451
|
+
struct.pack(
|
|
452
|
+
"<IHHHHHHIIIHHHHHII",
|
|
453
|
+
0x02014B50,
|
|
454
|
+
45,
|
|
455
|
+
45,
|
|
456
|
+
0x800,
|
|
457
|
+
0,
|
|
458
|
+
0,
|
|
459
|
+
33,
|
|
460
|
+
crc,
|
|
461
|
+
0xFFFFFFFF,
|
|
462
|
+
0xFFFFFFFF,
|
|
463
|
+
len(name),
|
|
464
|
+
len(extra),
|
|
465
|
+
0,
|
|
466
|
+
0,
|
|
467
|
+
0,
|
|
468
|
+
0,
|
|
469
|
+
0xFFFFFFFF,
|
|
470
|
+
)
|
|
471
|
+
+ name
|
|
472
|
+
+ extra
|
|
473
|
+
)
|
|
474
|
+
count += 1
|
|
475
|
+
central_offset, central_size = output.tell(), directory.tell()
|
|
476
|
+
directory.seek(0)
|
|
477
|
+
shutil.copyfileobj(directory, output, 1024 * 1024)
|
|
478
|
+
zip64_offset = output.tell()
|
|
479
|
+
output.write(
|
|
480
|
+
struct.pack(
|
|
481
|
+
"<IQHHIIQQQQ",
|
|
482
|
+
0x06064B50,
|
|
483
|
+
44,
|
|
484
|
+
45,
|
|
485
|
+
45,
|
|
486
|
+
0,
|
|
487
|
+
0,
|
|
488
|
+
count,
|
|
489
|
+
count,
|
|
490
|
+
central_size,
|
|
491
|
+
central_offset,
|
|
492
|
+
)
|
|
493
|
+
)
|
|
494
|
+
output.write(struct.pack("<IIQI", 0x07064B50, 0, zip64_offset, 1))
|
|
495
|
+
output.write(
|
|
496
|
+
struct.pack("<IHHHHIIH", 0x06054B50, 0, 0, 0xFFFF, 0xFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0)
|
|
497
|
+
)
|
|
498
|
+
|
|
499
|
+
|
|
500
|
+
class _ArchiveResponse(FileResponse):
|
|
501
|
+
def __init__(self, path, *, release, **kwargs):
|
|
502
|
+
super().__init__(path, **kwargs)
|
|
503
|
+
self.release = release
|
|
504
|
+
|
|
505
|
+
async def __call__(self, scope, receive, send):
|
|
506
|
+
try:
|
|
507
|
+
await super().__call__(scope, receive, send)
|
|
508
|
+
finally:
|
|
509
|
+
# Also release pins when a browser disconnects partway through a ZIP.
|
|
510
|
+
with CancelScope(shield=True):
|
|
511
|
+
await run_in_threadpool(self.release, self.path)
|
|
512
|
+
|
|
513
|
+
|
|
514
|
+
class Studio:
|
|
515
|
+
def __init__(
|
|
516
|
+
self,
|
|
517
|
+
config: Config,
|
|
518
|
+
output_root: Path | None = None,
|
|
519
|
+
*,
|
|
520
|
+
cache_mb: int = 256,
|
|
521
|
+
output_locations: dict[str, Path] | None = None,
|
|
522
|
+
):
|
|
523
|
+
if isinstance(cache_mb, bool) or not isinstance(cache_mb, int) or cache_mb < 1:
|
|
524
|
+
raise ValueError("Studio cache size must be a positive number of MiB")
|
|
525
|
+
self.cache_limit = cache_mb * 1024 * 1024
|
|
526
|
+
self.config = config.validate()
|
|
527
|
+
self.output_root = (output_root or config.path("dataset/ui")).resolve()
|
|
528
|
+
self.protected = (
|
|
529
|
+
config.path(config.data.raw),
|
|
530
|
+
config.path(config.data.processed),
|
|
531
|
+
*(config.path(path) for path in config.fonts.directories),
|
|
532
|
+
)
|
|
533
|
+
if output_locations is not None and (
|
|
534
|
+
not isinstance(output_locations, dict)
|
|
535
|
+
or len(output_locations) > 8
|
|
536
|
+
or any(
|
|
537
|
+
not isinstance(name, str) or not LOCATION.fullmatch(name) or name == "default"
|
|
538
|
+
for name in output_locations
|
|
539
|
+
)
|
|
540
|
+
or any(
|
|
541
|
+
not isinstance(path, (str, os.PathLike)) or not str(path).strip()
|
|
542
|
+
for path in output_locations.values()
|
|
543
|
+
)
|
|
544
|
+
):
|
|
545
|
+
raise ValueError("Use at most eight named output locations; 'default' is reserved")
|
|
546
|
+
self.output_locations = {
|
|
547
|
+
"default": self.output_root,
|
|
548
|
+
**{name: config.path(str(path)) for name, path in (output_locations or {}).items()},
|
|
549
|
+
}
|
|
550
|
+
roots = list(self.output_locations.values())
|
|
551
|
+
for index, root in enumerate(roots):
|
|
552
|
+
self._validate_output_root(root, roots[:index])
|
|
553
|
+
self.metadata = self.output_root / ".jobs"
|
|
554
|
+
self.cache = self.metadata / ".cache"
|
|
555
|
+
self.uploads = self.metadata / "uploads"
|
|
556
|
+
# Fonts added in the app stay in this workspace, never in the project's fonts/.
|
|
557
|
+
self.user_fonts = self.metadata / "fonts"
|
|
558
|
+
self._root_locks = []
|
|
559
|
+
try:
|
|
560
|
+
for root in roots:
|
|
561
|
+
root.mkdir(parents=True, exist_ok=True)
|
|
562
|
+
metadata = root / ".jobs"
|
|
563
|
+
if metadata.is_symlink():
|
|
564
|
+
raise ValueError("Job metadata directory must not be a symbolic link")
|
|
565
|
+
metadata.mkdir(exist_ok=True)
|
|
566
|
+
self._root_locks.append(_lock_output(metadata))
|
|
567
|
+
for path, label in (
|
|
568
|
+
(self.cache, "Studio cache"),
|
|
569
|
+
(self.uploads, "Upload directory"),
|
|
570
|
+
(self.user_fonts, "User font directory"),
|
|
571
|
+
):
|
|
572
|
+
if path.is_symlink():
|
|
573
|
+
raise ValueError(f"{label} must not be a symbolic link")
|
|
574
|
+
path.mkdir(exist_ok=True)
|
|
575
|
+
except BaseException:
|
|
576
|
+
for lock in reversed(self._root_locks):
|
|
577
|
+
lock.close()
|
|
578
|
+
raise
|
|
579
|
+
self.lock = threading.RLock()
|
|
580
|
+
self.upload_lock = threading.RLock()
|
|
581
|
+
self.upload_validation_lock = threading.Lock()
|
|
582
|
+
self.archive_lock = threading.RLock()
|
|
583
|
+
self.preview_lock = threading.Lock()
|
|
584
|
+
self.executor = ThreadPoolExecutor(max_workers=1, thread_name_prefix="synthocr-studio")
|
|
585
|
+
self.active: dict | None = None
|
|
586
|
+
self.cancel_event = threading.Event()
|
|
587
|
+
self._fonts = None
|
|
588
|
+
self._font_catalog = None
|
|
589
|
+
self._coverage = Coverage(min(config.fonts.coverage_cache_size, 128))
|
|
590
|
+
self._capacities = OrderedDict()
|
|
591
|
+
self._leased: dict[Path, int] = {}
|
|
592
|
+
self._uploading: set[str] = set()
|
|
593
|
+
self._claimed_uploads: set[str] = set()
|
|
594
|
+
self._used_uploads = OrderedDict()
|
|
595
|
+
try:
|
|
596
|
+
self._trim_uploads()
|
|
597
|
+
self._discard_legacy_cache()
|
|
598
|
+
self._trim_cache()
|
|
599
|
+
# Recover at most the displayed history, without loading a dataset-size index.
|
|
600
|
+
for path in self._recent_paths():
|
|
601
|
+
try:
|
|
602
|
+
job = self._load(path)
|
|
603
|
+
self.job_root(job)
|
|
604
|
+
if (
|
|
605
|
+
isinstance(job.get("upload"), dict)
|
|
606
|
+
and (self.dataset(job) / "source-upload.json").is_file()
|
|
607
|
+
):
|
|
608
|
+
self._used_uploads[job["upload"]["id"]] = job["name"]
|
|
609
|
+
except OSError, ValueError, HTTPException:
|
|
610
|
+
continue
|
|
611
|
+
if job.get("status") in ACTIVE:
|
|
612
|
+
job.update(
|
|
613
|
+
status="failed",
|
|
614
|
+
error="The app closed before generation finished. Start a new run.",
|
|
615
|
+
)
|
|
616
|
+
self._save(job)
|
|
617
|
+
except BaseException:
|
|
618
|
+
self.close()
|
|
619
|
+
raise
|
|
620
|
+
|
|
621
|
+
def _validate_output_root(self, root: Path, others):
|
|
622
|
+
if any(
|
|
623
|
+
path.name.casefold() == ".jobs" or self._is_dataset(path)
|
|
624
|
+
for path in (root, *root.parents)
|
|
625
|
+
):
|
|
626
|
+
raise ValueError("Choose an output root outside existing datasets and job metadata")
|
|
627
|
+
if any(self._roots_overlap(root, path) for path in self.protected):
|
|
628
|
+
raise ValueError("UI output root must be separate from raw, processed data and fonts")
|
|
629
|
+
if any(self._roots_overlap(root, other) for other in others):
|
|
630
|
+
raise ValueError("Output locations must not overlap or refer to the same folder")
|
|
631
|
+
|
|
632
|
+
@staticmethod
|
|
633
|
+
def _roots_overlap(first: Path, second: Path) -> bool:
|
|
634
|
+
if first.is_relative_to(second) or second.is_relative_to(first):
|
|
635
|
+
return True
|
|
636
|
+
# Case-insensitive volumes may expose the same folder through differently
|
|
637
|
+
# cased POSIX paths. Compare existing ancestors by filesystem identity.
|
|
638
|
+
for root, other in ((first, second), (second, first)):
|
|
639
|
+
if not root.exists():
|
|
640
|
+
continue
|
|
641
|
+
for candidate in (other, *other.parents):
|
|
642
|
+
if candidate.exists() and root.samefile(candidate):
|
|
643
|
+
return True
|
|
644
|
+
return False
|
|
645
|
+
|
|
646
|
+
def _discard_legacy_cache(self):
|
|
647
|
+
# Earlier studio versions retained a second export tree and ZIP for every
|
|
648
|
+
# format. These directories contain only disposable studio derivatives.
|
|
649
|
+
with os.scandir(self.metadata) as entries:
|
|
650
|
+
for entry in entries:
|
|
651
|
+
if JOB_ID.fullmatch(entry.name) and entry.is_dir(follow_symlinks=False):
|
|
652
|
+
shutil.rmtree(entry.path)
|
|
653
|
+
elif entry.name.startswith(".export-") and entry.is_dir(follow_symlinks=False):
|
|
654
|
+
shutil.rmtree(entry.path)
|
|
655
|
+
with os.scandir(self.cache) as entries:
|
|
656
|
+
for entry in entries:
|
|
657
|
+
if entry.name.endswith(".tmp") and entry.is_file(follow_symlinks=False):
|
|
658
|
+
Path(entry.path).unlink(missing_ok=True)
|
|
659
|
+
|
|
660
|
+
def _cache_files(self):
|
|
661
|
+
if self.cache.is_symlink():
|
|
662
|
+
raise HTTPException(400, "Invalid studio cache")
|
|
663
|
+
with os.scandir(self.cache) as entries:
|
|
664
|
+
for entry in entries:
|
|
665
|
+
if entry.is_file(follow_symlinks=False):
|
|
666
|
+
yield Path(entry.path)
|
|
667
|
+
|
|
668
|
+
def _trim_cache(self, reserve=0):
|
|
669
|
+
"""Evict only disposable derivatives; never remove original datasets."""
|
|
670
|
+
if reserve > self.cache_limit:
|
|
671
|
+
raise HTTPException(
|
|
672
|
+
413,
|
|
673
|
+
"This download exceeds the studio cache limit. Download the manifest, "
|
|
674
|
+
"use the dataset in the output folder, or restart with a larger --cache-mb.",
|
|
675
|
+
)
|
|
676
|
+
while True:
|
|
677
|
+
total, count, oldest = 0, 0, None
|
|
678
|
+
for path in self._cache_files():
|
|
679
|
+
info = path.stat()
|
|
680
|
+
total += info.st_size
|
|
681
|
+
count += 1
|
|
682
|
+
if path not in self._leased and (oldest is None or info.st_mtime_ns < oldest[0]):
|
|
683
|
+
oldest = (info.st_mtime_ns, path)
|
|
684
|
+
if total + reserve <= self.cache_limit and count < CACHE_FILES:
|
|
685
|
+
return total
|
|
686
|
+
if oldest is None:
|
|
687
|
+
raise HTTPException(
|
|
688
|
+
503, "Download cache is busy; retry after current downloads finish"
|
|
689
|
+
)
|
|
690
|
+
oldest[1].unlink(missing_ok=True)
|
|
691
|
+
|
|
692
|
+
def storage(self):
|
|
693
|
+
with self.archive_lock:
|
|
694
|
+
return {
|
|
695
|
+
"cache_limit_bytes": self.cache_limit,
|
|
696
|
+
"current_cache_bytes": sum(path.stat().st_size for path in self._cache_files()),
|
|
697
|
+
"policy": "Least recently used downloads are removed; datasets are retained",
|
|
698
|
+
"upload_limit_bytes": UPLOAD_BYTES,
|
|
699
|
+
}
|
|
700
|
+
|
|
701
|
+
def location_root(self, location: str, expected: str | None = None) -> Path:
|
|
702
|
+
if not isinstance(location, str) or location not in self.output_locations:
|
|
703
|
+
raise HTTPException(404, "Output location is not available")
|
|
704
|
+
root = self.output_locations[location]
|
|
705
|
+
if expected is not None and expected != str(root):
|
|
706
|
+
raise HTTPException(
|
|
707
|
+
409, "This dataset's output location has changed; restore its original location"
|
|
708
|
+
)
|
|
709
|
+
if root.is_symlink() or root.resolve() != root or not root.is_dir():
|
|
710
|
+
raise HTTPException(400, "Output location is not an accessible ordinary directory")
|
|
711
|
+
return root
|
|
712
|
+
|
|
713
|
+
def job_root(self, job) -> Path:
|
|
714
|
+
location = job.get("output_location", "default")
|
|
715
|
+
expected = job.get("output_root")
|
|
716
|
+
if location != "default" and not isinstance(expected, str):
|
|
717
|
+
raise HTTPException(400, "Dataset output location identity is missing")
|
|
718
|
+
return self.location_root(location, expected)
|
|
719
|
+
|
|
720
|
+
@staticmethod
|
|
721
|
+
def _is_dataset(path: Path) -> bool:
|
|
722
|
+
return any(
|
|
723
|
+
(path / name).exists() or (path / name).is_symlink()
|
|
724
|
+
for name in ("run.json", "manifest.jsonl")
|
|
725
|
+
)
|
|
726
|
+
|
|
727
|
+
def directory(self, location: str, path: str = "", expected: str | None = None) -> Path:
|
|
728
|
+
root = self.location_root(location, expected)
|
|
729
|
+
parts = _folder_parts(path)
|
|
730
|
+
# Generation creates the image directory before writing run/manifest
|
|
731
|
+
# markers. Reserve the entire active destination during that interval,
|
|
732
|
+
# including queued output that does not yet exist. Derive its path from
|
|
733
|
+
# recorded components; dataset() would recurse through directory().
|
|
734
|
+
reserved = None
|
|
735
|
+
active = self.active
|
|
736
|
+
if active and active["status"] in ACTIVE:
|
|
737
|
+
active_root = self.output_locations.get(active.get("output_location", "default"))
|
|
738
|
+
name = active.get("name")
|
|
739
|
+
if (
|
|
740
|
+
active_root
|
|
741
|
+
and isinstance(name, str)
|
|
742
|
+
and NAME.fullmatch(name)
|
|
743
|
+
and _folder_name(name)
|
|
744
|
+
):
|
|
745
|
+
reserved = active_root.joinpath(
|
|
746
|
+
*_folder_parts(active.get("output_folder", "")), name
|
|
747
|
+
)
|
|
748
|
+
current = root
|
|
749
|
+
for index in range(len(parts) + 1):
|
|
750
|
+
if reserved is not None and current.is_relative_to(reserved):
|
|
751
|
+
raise HTTPException(409, "That folder is reserved for the active dataset")
|
|
752
|
+
if current.is_symlink() or current.resolve() != current or not current.is_dir():
|
|
753
|
+
raise HTTPException(400, "Choose an existing folder without symbolic links")
|
|
754
|
+
if self._is_dataset(current):
|
|
755
|
+
raise HTTPException(400, "Choose a folder outside existing datasets")
|
|
756
|
+
if any(current.is_relative_to(protected) for protected in self.protected):
|
|
757
|
+
raise HTTPException(
|
|
758
|
+
400, "Output folders must stay separate from source data and fonts"
|
|
759
|
+
)
|
|
760
|
+
if index < len(parts):
|
|
761
|
+
current = current / parts[index]
|
|
762
|
+
return current
|
|
763
|
+
|
|
764
|
+
def directories(self, location: str = "default", path: str = ""):
|
|
765
|
+
parent = self.directory(location, path)
|
|
766
|
+
children, truncated = [], False
|
|
767
|
+
with os.scandir(parent) as entries:
|
|
768
|
+
for scanned, entry in enumerate(entries):
|
|
769
|
+
if scanned >= DIRECTORY_SCAN_LIMIT:
|
|
770
|
+
truncated = True
|
|
771
|
+
break
|
|
772
|
+
if not _folder_name(entry.name) or not entry.is_dir(follow_symlinks=False):
|
|
773
|
+
continue
|
|
774
|
+
relative = "/".join(filter(None, (path, entry.name)))
|
|
775
|
+
try:
|
|
776
|
+
self.directory(location, relative)
|
|
777
|
+
except OSError, ValueError, HTTPException:
|
|
778
|
+
continue
|
|
779
|
+
if len(children) >= DIRECTORY_LIMIT:
|
|
780
|
+
truncated = True
|
|
781
|
+
break
|
|
782
|
+
children.append({"name": entry.name, "path": relative})
|
|
783
|
+
children.sort(key=lambda entry: entry["name"].casefold())
|
|
784
|
+
return {
|
|
785
|
+
"location": location,
|
|
786
|
+
"path": path,
|
|
787
|
+
"parent": path.rpartition("/")[0] if path else None,
|
|
788
|
+
"directories": children,
|
|
789
|
+
"truncated": truncated,
|
|
790
|
+
"root": str(self.output_locations[location]),
|
|
791
|
+
"absolute_path": str(parent),
|
|
792
|
+
}
|
|
793
|
+
|
|
794
|
+
def create_directory(self, request: DirectoryRequest):
|
|
795
|
+
with self.lock:
|
|
796
|
+
relative = "/".join(filter(None, (request.path, request.name)))
|
|
797
|
+
_folder_parts(relative)
|
|
798
|
+
parent = self.directory(request.location, request.path)
|
|
799
|
+
target = parent / request.name
|
|
800
|
+
if target.exists() or target.is_symlink():
|
|
801
|
+
raise HTTPException(
|
|
802
|
+
409, "That folder already exists; choose it or use another name"
|
|
803
|
+
)
|
|
804
|
+
if (
|
|
805
|
+
self.active
|
|
806
|
+
and self.active["status"] in ACTIVE
|
|
807
|
+
and target == self.dataset(self.active)
|
|
808
|
+
):
|
|
809
|
+
raise HTTPException(409, "That folder is reserved for the active dataset")
|
|
810
|
+
target.mkdir()
|
|
811
|
+
return self.directories(request.location, relative)
|
|
812
|
+
|
|
813
|
+
def _pending_uploads(self):
|
|
814
|
+
if self.uploads.is_symlink():
|
|
815
|
+
raise HTTPException(400, "Invalid upload directory")
|
|
816
|
+
with os.scandir(self.uploads) as entries:
|
|
817
|
+
for entry in entries:
|
|
818
|
+
if JOB_ID.fullmatch(entry.name) and entry.is_dir(follow_symlinks=False):
|
|
819
|
+
yield Path(entry.path)
|
|
820
|
+
|
|
821
|
+
def _trim_uploads(self):
|
|
822
|
+
"""Expire unused inputs only; retained dataset sources are never removed."""
|
|
823
|
+
with self.upload_lock:
|
|
824
|
+
now = time.time()
|
|
825
|
+
for path in self._pending_uploads():
|
|
826
|
+
if path.name in self._uploading | self._claimed_uploads:
|
|
827
|
+
continue
|
|
828
|
+
if not (path / "upload.json").is_file() or now - path.stat().st_mtime > UPLOAD_TTL:
|
|
829
|
+
shutil.rmtree(path)
|
|
830
|
+
|
|
831
|
+
def _upload_usage(self):
|
|
832
|
+
size, count = 0, 0
|
|
833
|
+
for path in self._pending_uploads():
|
|
834
|
+
count += 1
|
|
835
|
+
with os.scandir(path) as entries:
|
|
836
|
+
for entry in entries:
|
|
837
|
+
if entry.name.startswith("source.") and entry.is_file(follow_symlinks=False):
|
|
838
|
+
size += entry.stat(follow_symlinks=False).st_size
|
|
839
|
+
return size, count
|
|
840
|
+
|
|
841
|
+
def _begin_upload(self, filename, language):
|
|
842
|
+
if (
|
|
843
|
+
not filename
|
|
844
|
+
or len(filename.encode("utf-8")) > 255
|
|
845
|
+
or Path(filename).name != filename
|
|
846
|
+
or "\\" in filename
|
|
847
|
+
or any(ord(char) < 32 for char in filename)
|
|
848
|
+
):
|
|
849
|
+
raise HTTPException(422, "Choose a TXT, JSONL or CSV file")
|
|
850
|
+
suffix = Path(filename).suffix.lower()
|
|
851
|
+
if suffix not in {".txt", ".jsonl", ".csv"}:
|
|
852
|
+
raise HTTPException(422, "Upload a UTF-8 TXT, JSONL or CSV file")
|
|
853
|
+
if len(language) > 128 or (language not in LANGUAGES and not LANGUAGE.fullmatch(language)):
|
|
854
|
+
raise HTTPException(422, "Choose a source language for the uploaded file")
|
|
855
|
+
with self.upload_lock:
|
|
856
|
+
self._trim_uploads()
|
|
857
|
+
size, count = self._upload_usage()
|
|
858
|
+
if count >= UPLOAD_FILES or size >= UPLOAD_TOTAL_BYTES:
|
|
859
|
+
raise HTTPException(
|
|
860
|
+
413, "Pending uploads are full. Remove an unused upload or generate its dataset"
|
|
861
|
+
)
|
|
862
|
+
identifier = uuid4().hex
|
|
863
|
+
path = self.uploads / identifier
|
|
864
|
+
path.mkdir()
|
|
865
|
+
self._uploading.add(identifier)
|
|
866
|
+
return identifier, path / f"source{suffix}"
|
|
867
|
+
|
|
868
|
+
def _write_upload_chunk(self, stream, chunk):
|
|
869
|
+
with self.upload_lock:
|
|
870
|
+
if stream.tell() + len(chunk) > UPLOAD_BYTES:
|
|
871
|
+
raise HTTPException(413, "Uploaded files must be at most 50 MiB")
|
|
872
|
+
size, _ = self._upload_usage()
|
|
873
|
+
if size + len(chunk) > UPLOAD_TOTAL_BYTES:
|
|
874
|
+
raise HTTPException(413, "Pending uploads exceed the 100 MiB storage limit")
|
|
875
|
+
stream.write(chunk)
|
|
876
|
+
# Make accepted bytes visible to quota checks for simultaneous uploads.
|
|
877
|
+
stream.flush()
|
|
878
|
+
|
|
879
|
+
def _validate_upload(self, identifier, path, filename, language, selected_languages=None):
|
|
880
|
+
with (
|
|
881
|
+
self.upload_validation_lock,
|
|
882
|
+
tempfile.TemporaryDirectory(prefix="synthocr-upload-index-") as directory,
|
|
883
|
+
):
|
|
884
|
+
db = sqlite3.connect(Path(directory) / "labels.sqlite3")
|
|
885
|
+
try:
|
|
886
|
+
db.execute("PRAGMA cache_size=-2048")
|
|
887
|
+
db.execute("PRAGMA temp_store=FILE")
|
|
888
|
+
db.execute("PRAGMA journal_mode=OFF")
|
|
889
|
+
db.execute(
|
|
890
|
+
"CREATE TABLE labels (id INTEGER PRIMARY KEY, language TEXT, text TEXT, "
|
|
891
|
+
"genre TEXT, UNIQUE(language,text))"
|
|
892
|
+
)
|
|
893
|
+
return self._inspect_upload(
|
|
894
|
+
identifier, path, filename, language, db, selected_languages
|
|
895
|
+
)
|
|
896
|
+
finally:
|
|
897
|
+
db.close()
|
|
898
|
+
|
|
899
|
+
def _inspect_upload(self, identifier, path, filename, language, db, selected_languages=None):
|
|
900
|
+
selection_languages = _language_tags(language, selected_languages)
|
|
901
|
+
selected = set(selection_languages)
|
|
902
|
+
excluded = {*self.config.generation.exclude_genres, "spelling"}
|
|
903
|
+
fallback = selection_languages
|
|
904
|
+
fallback = fallback[0] if len(fallback) == 1 else ""
|
|
905
|
+
count, unlabelled, languages = 0, 0, set()
|
|
906
|
+
for line, row in records(path, path.parent, self.config.data.max_line_chars):
|
|
907
|
+
value = row.get("text")
|
|
908
|
+
if not isinstance(value, str):
|
|
909
|
+
raise ValueError(f"Line {line}: text must be a string")
|
|
910
|
+
value.encode("utf-8", errors="strict")
|
|
911
|
+
text = normalize(value, "NFC")
|
|
912
|
+
if not required_codepoints(text):
|
|
913
|
+
continue
|
|
914
|
+
tag = row.get("language") or ""
|
|
915
|
+
if not isinstance(tag, str) or len(tag) > 128:
|
|
916
|
+
raise ValueError(f"Line {line}: language must be a language tag")
|
|
917
|
+
if not isinstance(row.get("attribution", {}), dict):
|
|
918
|
+
raise ValueError(f"Line {line}: attribution must be an object")
|
|
919
|
+
genre = row.get("genre") or path.stem
|
|
920
|
+
if not isinstance(genre, str):
|
|
921
|
+
raise ValueError(f"Line {line}: genre must be a string")
|
|
922
|
+
if not tag:
|
|
923
|
+
unlabelled += 1
|
|
924
|
+
if not fallback:
|
|
925
|
+
raise HTTPException(
|
|
926
|
+
422,
|
|
927
|
+
"This file has text without language tags. Choose one explicit language, "
|
|
928
|
+
"or add a language field to every JSONL/CSV record",
|
|
929
|
+
)
|
|
930
|
+
tag = fallback
|
|
931
|
+
tag = resolve_source_language(tag, row.get("attribution", {}))
|
|
932
|
+
tag = canonical_language(tag)
|
|
933
|
+
languages.add(tag)
|
|
934
|
+
if len(languages) > 128:
|
|
935
|
+
raise HTTPException(422, "An uploaded file may contain at most 128 language tags")
|
|
936
|
+
count += 1
|
|
937
|
+
for label in chunks(text, self.config.data.max_chars):
|
|
938
|
+
if required_codepoints(label):
|
|
939
|
+
db.execute(
|
|
940
|
+
"INSERT OR IGNORE INTO labels(language,text,genre) VALUES(?,?,?)",
|
|
941
|
+
(tag, label, genre),
|
|
942
|
+
)
|
|
943
|
+
# Shared preparation keeps the first origin, except that
|
|
944
|
+
# natural text replaces an identical spelling diagnostic.
|
|
945
|
+
if genre != "spelling":
|
|
946
|
+
db.execute(
|
|
947
|
+
"UPDATE labels SET genre=? WHERE language=? AND text=? "
|
|
948
|
+
"AND genre='spelling'",
|
|
949
|
+
(genre, tag, label),
|
|
950
|
+
)
|
|
951
|
+
if not count:
|
|
952
|
+
raise HTTPException(422, "The file contains no visible text or numbers")
|
|
953
|
+
conditions, parameters = [], []
|
|
954
|
+
if selected:
|
|
955
|
+
conditions.append(f"language IN ({','.join('?' for _ in selected)})")
|
|
956
|
+
parameters.extend(sorted(selected))
|
|
957
|
+
if excluded:
|
|
958
|
+
conditions.append(f"genre NOT IN ({','.join('?' for _ in excluded)})")
|
|
959
|
+
parameters.extend(sorted(excluded))
|
|
960
|
+
selection = " WHERE " + " AND ".join(conditions) if conditions else ""
|
|
961
|
+
selected_records = db.execute(
|
|
962
|
+
"SELECT COUNT(*) FROM labels" + selection, parameters
|
|
963
|
+
).fetchone()[0]
|
|
964
|
+
if not selected_records:
|
|
965
|
+
raise HTTPException(
|
|
966
|
+
422,
|
|
967
|
+
"The uploaded file has no usable text for the selected language. "
|
|
968
|
+
"Check its language tags and excluded genres",
|
|
969
|
+
)
|
|
970
|
+
preview_language, first_label = db.execute(
|
|
971
|
+
"SELECT language,text FROM labels" + selection + " ORDER BY id LIMIT 1", parameters
|
|
972
|
+
).fetchone()
|
|
973
|
+
preview = next(chunks(first_label, min(self.config.data.max_chars, 200)), "")
|
|
974
|
+
document = {
|
|
975
|
+
"id": identifier,
|
|
976
|
+
"filename": filename,
|
|
977
|
+
"stored_filename": path.name,
|
|
978
|
+
"bytes": path.stat().st_size,
|
|
979
|
+
"sha256": sha256(path),
|
|
980
|
+
"records": selected_records,
|
|
981
|
+
"source_records": count,
|
|
982
|
+
"languages": sorted(languages),
|
|
983
|
+
"selected_languages": selection_languages,
|
|
984
|
+
"unlabelled_records": unlabelled,
|
|
985
|
+
"language": language,
|
|
986
|
+
"preview_text": preview,
|
|
987
|
+
"preview_language": preview_language,
|
|
988
|
+
"created_at": _now(),
|
|
989
|
+
}
|
|
990
|
+
_atomic_json(path.parent / "upload.json", document)
|
|
991
|
+
return document
|
|
992
|
+
|
|
993
|
+
async def upload(self, request, filename, language, languages=None):
|
|
994
|
+
try:
|
|
995
|
+
_language_tags(language, languages)
|
|
996
|
+
except ValueError as error:
|
|
997
|
+
raise HTTPException(422, str(error)) from error
|
|
998
|
+
identifier, path = await run_in_threadpool(self._begin_upload, filename, language)
|
|
999
|
+
successful = False
|
|
1000
|
+
try:
|
|
1001
|
+
with path.open("xb", buffering=0) as stream:
|
|
1002
|
+
async for chunk in request.stream():
|
|
1003
|
+
# ASGI controls incoming chunk size; disk writes and quota checks
|
|
1004
|
+
# stay bounded even when a client sends the entire body at once.
|
|
1005
|
+
view = memoryview(chunk)
|
|
1006
|
+
for start in range(0, len(view), UPLOAD_CHUNK):
|
|
1007
|
+
await run_in_threadpool(
|
|
1008
|
+
self._write_upload_chunk, stream, view[start : start + UPLOAD_CHUNK]
|
|
1009
|
+
)
|
|
1010
|
+
arguments = (identifier, path, filename, language)
|
|
1011
|
+
result = await run_in_threadpool(
|
|
1012
|
+
self._validate_upload, *arguments, *([languages] if languages is not None else [])
|
|
1013
|
+
)
|
|
1014
|
+
if await request.is_disconnected():
|
|
1015
|
+
raise HTTPException(499, "Upload cancelled")
|
|
1016
|
+
successful = True
|
|
1017
|
+
return result
|
|
1018
|
+
finally:
|
|
1019
|
+
with self.upload_lock:
|
|
1020
|
+
self._uploading.discard(identifier)
|
|
1021
|
+
if not successful:
|
|
1022
|
+
shutil.rmtree(path.parent, ignore_errors=True)
|
|
1023
|
+
|
|
1024
|
+
def uploaded(self, identifier):
|
|
1025
|
+
if not JOB_ID.fullmatch(identifier):
|
|
1026
|
+
raise HTTPException(404, "Upload not found; choose the file again")
|
|
1027
|
+
with self.upload_lock:
|
|
1028
|
+
if identifier in self._used_uploads:
|
|
1029
|
+
raise HTTPException(
|
|
1030
|
+
409,
|
|
1031
|
+
f"This file was saved with dataset {self._used_uploads[identifier]}. "
|
|
1032
|
+
"Choose the file again to generate another dataset",
|
|
1033
|
+
)
|
|
1034
|
+
self._trim_uploads()
|
|
1035
|
+
directory = self.uploads / identifier
|
|
1036
|
+
metadata = directory / "upload.json"
|
|
1037
|
+
try:
|
|
1038
|
+
if (
|
|
1039
|
+
directory.is_symlink()
|
|
1040
|
+
or metadata.is_symlink()
|
|
1041
|
+
or metadata.stat().st_size > 65536
|
|
1042
|
+
):
|
|
1043
|
+
raise ValueError("Invalid upload metadata")
|
|
1044
|
+
upload = json.loads(metadata.read_text(encoding="utf-8"))
|
|
1045
|
+
filename = upload["stored_filename"]
|
|
1046
|
+
if filename not in {"source.txt", "source.csv", "source.jsonl"}:
|
|
1047
|
+
raise ValueError("Invalid upload file")
|
|
1048
|
+
path = directory / filename
|
|
1049
|
+
if path.is_symlink() or not path.is_file() or upload["id"] != identifier:
|
|
1050
|
+
raise ValueError("Invalid upload source")
|
|
1051
|
+
except (OSError, ValueError, TypeError, KeyError) as error:
|
|
1052
|
+
raise HTTPException(404, "Upload not found; choose the file again") from error
|
|
1053
|
+
return upload, path
|
|
1054
|
+
|
|
1055
|
+
def remove_upload(self, identifier):
|
|
1056
|
+
with self.upload_lock:
|
|
1057
|
+
self.uploaded(identifier)
|
|
1058
|
+
if identifier in self._claimed_uploads:
|
|
1059
|
+
raise HTTPException(409, "This file is being used by the current generation")
|
|
1060
|
+
shutil.rmtree(self.uploads / identifier)
|
|
1061
|
+
return {"removed": True}
|
|
1062
|
+
|
|
1063
|
+
def _release_archive(self, path):
|
|
1064
|
+
with self.archive_lock:
|
|
1065
|
+
self._leased[path] -= 1
|
|
1066
|
+
if not self._leased[path]:
|
|
1067
|
+
del self._leased[path]
|
|
1068
|
+
|
|
1069
|
+
def _recent_paths(self):
|
|
1070
|
+
def candidates():
|
|
1071
|
+
with os.scandir(self.metadata) as entries:
|
|
1072
|
+
for entry in entries:
|
|
1073
|
+
if (
|
|
1074
|
+
entry.name.endswith(".json")
|
|
1075
|
+
and JOB_ID.fullmatch(entry.name[:-5])
|
|
1076
|
+
and entry.is_file(follow_symlinks=False)
|
|
1077
|
+
):
|
|
1078
|
+
yield Path(entry.path)
|
|
1079
|
+
|
|
1080
|
+
return heapq.nlargest(
|
|
1081
|
+
20,
|
|
1082
|
+
candidates(),
|
|
1083
|
+
key=lambda p: p.stat().st_mtime_ns,
|
|
1084
|
+
)
|
|
1085
|
+
|
|
1086
|
+
def _load(self, path):
|
|
1087
|
+
if path.is_symlink() or path.stat().st_size > 64 * 1024:
|
|
1088
|
+
raise ValueError("Invalid job metadata")
|
|
1089
|
+
job = json.loads(path.read_text(encoding="utf-8"))
|
|
1090
|
+
if (
|
|
1091
|
+
not isinstance(job, dict)
|
|
1092
|
+
or job.get("id") != path.stem
|
|
1093
|
+
or not isinstance(job.get("name"), str)
|
|
1094
|
+
or not NAME.fullmatch(job["name"])
|
|
1095
|
+
or not isinstance(job.get("status"), str)
|
|
1096
|
+
or job.get("status") not in ACTIVE | {"completed", "cancelled", "failed"}
|
|
1097
|
+
):
|
|
1098
|
+
raise ValueError("Invalid job metadata")
|
|
1099
|
+
return job
|
|
1100
|
+
|
|
1101
|
+
def _save(self, job):
|
|
1102
|
+
job["updated_at"] = _now()
|
|
1103
|
+
_atomic_json(self.metadata / f"{job['id']}.json", job)
|
|
1104
|
+
|
|
1105
|
+
def job(self, identifier):
|
|
1106
|
+
if not JOB_ID.fullmatch(identifier):
|
|
1107
|
+
raise HTTPException(404, "Job not found")
|
|
1108
|
+
with self.lock:
|
|
1109
|
+
if self.active and self.active["id"] == identifier:
|
|
1110
|
+
return self.active.copy()
|
|
1111
|
+
path = self.metadata / f"{identifier}.json"
|
|
1112
|
+
try:
|
|
1113
|
+
job = self._load(path)
|
|
1114
|
+
except (OSError, ValueError, KeyError) as error:
|
|
1115
|
+
raise HTTPException(404, "Job not found") from error
|
|
1116
|
+
if job.get("id") != identifier or not NAME.fullmatch(job.get("name", "")):
|
|
1117
|
+
raise HTTPException(404, "Job not found")
|
|
1118
|
+
self.job_root(job)
|
|
1119
|
+
if job.get("status") in ACTIVE:
|
|
1120
|
+
job.update(
|
|
1121
|
+
status="failed",
|
|
1122
|
+
error="The app closed before generation finished. Start a new run.",
|
|
1123
|
+
)
|
|
1124
|
+
self._save(job)
|
|
1125
|
+
return job
|
|
1126
|
+
|
|
1127
|
+
def jobs(self):
|
|
1128
|
+
with self.lock:
|
|
1129
|
+
result = []
|
|
1130
|
+
for path in self._recent_paths():
|
|
1131
|
+
try:
|
|
1132
|
+
result.append(self.job(path.stem))
|
|
1133
|
+
except HTTPException:
|
|
1134
|
+
continue
|
|
1135
|
+
return result
|
|
1136
|
+
|
|
1137
|
+
def dataset(self, job):
|
|
1138
|
+
root = self.job_root(job)
|
|
1139
|
+
parent = self.directory(
|
|
1140
|
+
job.get("output_location", "default"), job.get("output_folder", ""), str(root)
|
|
1141
|
+
)
|
|
1142
|
+
if not NAME.fullmatch(job["name"]) or not _folder_name(job["name"]):
|
|
1143
|
+
raise HTTPException(400, "Invalid dataset name")
|
|
1144
|
+
path = parent / job["name"]
|
|
1145
|
+
if path.is_symlink() or path.resolve().parent != parent:
|
|
1146
|
+
raise HTTPException(400, "Dataset path must stay inside the configured output root")
|
|
1147
|
+
if any(
|
|
1148
|
+
(path / name).is_symlink()
|
|
1149
|
+
for name in (
|
|
1150
|
+
"manifest.jsonl",
|
|
1151
|
+
"origins.jsonl",
|
|
1152
|
+
"run.json",
|
|
1153
|
+
"stats.json",
|
|
1154
|
+
"images",
|
|
1155
|
+
"source.jsonl",
|
|
1156
|
+
"source.txt",
|
|
1157
|
+
"source.csv",
|
|
1158
|
+
"source-upload.json",
|
|
1159
|
+
)
|
|
1160
|
+
):
|
|
1161
|
+
raise HTTPException(400, "Dataset artifacts must not be symbolic links")
|
|
1162
|
+
return path
|
|
1163
|
+
|
|
1164
|
+
def font_directories(self):
|
|
1165
|
+
return [*(self.config.path(p) for p in self.config.fonts.directories), self.user_fonts]
|
|
1166
|
+
|
|
1167
|
+
def fonts(self):
|
|
1168
|
+
with self.lock:
|
|
1169
|
+
if self._fonts is None:
|
|
1170
|
+
self._fonts = discover(self.font_directories())
|
|
1171
|
+
return self._fonts
|
|
1172
|
+
|
|
1173
|
+
def font_catalog(self):
|
|
1174
|
+
"""Every usable face with a readable name and the page languages it draws."""
|
|
1175
|
+
with self.lock:
|
|
1176
|
+
if self._font_catalog is None:
|
|
1177
|
+
fonts = self.fonts()
|
|
1178
|
+
ids = font_ids(fonts)
|
|
1179
|
+
user = self.user_fonts.resolve()
|
|
1180
|
+
entries = [
|
|
1181
|
+
{
|
|
1182
|
+
"id": ids[font],
|
|
1183
|
+
"name": _font_display_name(font),
|
|
1184
|
+
"file": Path(font.path).name,
|
|
1185
|
+
"user": Path(font.path).parent == user,
|
|
1186
|
+
"languages": [
|
|
1187
|
+
tag
|
|
1188
|
+
for tag, sample in LANGUAGE_SAMPLES.items()
|
|
1189
|
+
if not self._coverage.missing(font, required_codepoints(sample))
|
|
1190
|
+
],
|
|
1191
|
+
}
|
|
1192
|
+
for font in fonts
|
|
1193
|
+
]
|
|
1194
|
+
entries.sort(key=lambda entry: (entry["name"].casefold(), entry["id"]))
|
|
1195
|
+
self._font_catalog = (entries, ids)
|
|
1196
|
+
return self._font_catalog
|
|
1197
|
+
|
|
1198
|
+
def selected_fonts(self, include):
|
|
1199
|
+
fonts = self.fonts()
|
|
1200
|
+
if not include:
|
|
1201
|
+
return fonts
|
|
1202
|
+
try:
|
|
1203
|
+
return select(fonts, include, self.font_catalog()[1])
|
|
1204
|
+
except ValueError as error:
|
|
1205
|
+
raise HTTPException(422, str(error)) from error
|
|
1206
|
+
|
|
1207
|
+
def _forget_fonts(self):
|
|
1208
|
+
self._fonts = None
|
|
1209
|
+
self._font_catalog = None
|
|
1210
|
+
|
|
1211
|
+
async def add_font(self, request, filename):
|
|
1212
|
+
suffix = Path(filename).suffix.lower()
|
|
1213
|
+
if suffix not in FONT_SUFFIXES or not _folder_name(Path(filename).name):
|
|
1214
|
+
raise HTTPException(422, "Choose a TTF, OTF, TTC or OTC font file")
|
|
1215
|
+
# Not a font suffix, so discovery never sees a partial upload.
|
|
1216
|
+
temporary = self.user_fonts / f".upload-{uuid4().hex}.part"
|
|
1217
|
+
try:
|
|
1218
|
+
size = 0
|
|
1219
|
+
with temporary.open("xb") as stream:
|
|
1220
|
+
async for chunk in request.stream():
|
|
1221
|
+
size += len(chunk)
|
|
1222
|
+
if size > FONT_UPLOAD_BYTES:
|
|
1223
|
+
raise HTTPException(413, "Font files must be at most 32 MiB")
|
|
1224
|
+
await run_in_threadpool(stream.write, chunk)
|
|
1225
|
+
return await run_in_threadpool(self._install_font, temporary, filename, suffix)
|
|
1226
|
+
finally:
|
|
1227
|
+
temporary.unlink(missing_ok=True)
|
|
1228
|
+
|
|
1229
|
+
def _install_font(self, temporary, filename, suffix):
|
|
1230
|
+
if not temporary.stat().st_size:
|
|
1231
|
+
raise HTTPException(422, "The font file is empty")
|
|
1232
|
+
try:
|
|
1233
|
+
faces = 1
|
|
1234
|
+
if suffix in {".ttc", ".otc"}:
|
|
1235
|
+
with TTCollection(temporary, lazy=True) as collection:
|
|
1236
|
+
faces = len(collection.fonts)
|
|
1237
|
+
for index in range(faces):
|
|
1238
|
+
ImageFont.truetype(str(temporary), 12, index=index)
|
|
1239
|
+
except (OSError, ValueError, TTLibError) as error:
|
|
1240
|
+
raise HTTPException(422, "This file is not a usable font") from error
|
|
1241
|
+
digest = sha256(temporary)
|
|
1242
|
+
with self.lock:
|
|
1243
|
+
entries, _ = self.font_catalog()
|
|
1244
|
+
existing = [entry for entry in entries if entry["id"].startswith(f"{digest}:")]
|
|
1245
|
+
if existing:
|
|
1246
|
+
return {"fonts": existing, "added": False}
|
|
1247
|
+
if sum(1 for _ in self.user_fonts.glob("*")) >= USER_FONT_LIMIT:
|
|
1248
|
+
raise HTTPException(409, f"At most {USER_FONT_LIMIT} fonts can be added")
|
|
1249
|
+
stem = re.sub(r"[^A-Za-z0-9._-]+", "-", Path(filename).stem).strip("-.")[:60]
|
|
1250
|
+
os.replace(temporary, self.user_fonts / f"{stem or 'font'}-{digest[:12]}{suffix}")
|
|
1251
|
+
self._forget_fonts()
|
|
1252
|
+
entries, _ = self.font_catalog()
|
|
1253
|
+
added = [entry for entry in entries if entry["id"].startswith(f"{digest}:")]
|
|
1254
|
+
return {"fonts": added, "added": True}
|
|
1255
|
+
|
|
1256
|
+
def remove_font(self, digest):
|
|
1257
|
+
with self.lock:
|
|
1258
|
+
if self.active and self.active["status"] in ACTIVE:
|
|
1259
|
+
raise HTTPException(409, "Wait for the current generation to finish")
|
|
1260
|
+
entries, _ = self.font_catalog()
|
|
1261
|
+
match = next(
|
|
1262
|
+
(e for e in entries if e["user"] and e["id"].startswith(f"{digest}:")), None
|
|
1263
|
+
)
|
|
1264
|
+
if match is None:
|
|
1265
|
+
raise HTTPException(404, "Only fonts you added can be removed")
|
|
1266
|
+
path = self.user_fonts / match["file"]
|
|
1267
|
+
if path.is_symlink() or path.parent.resolve() != self.user_fonts.resolve():
|
|
1268
|
+
raise HTTPException(400, "Invalid font file")
|
|
1269
|
+
path.unlink()
|
|
1270
|
+
self._forget_fonts()
|
|
1271
|
+
return {"removed": digest}
|
|
1272
|
+
|
|
1273
|
+
def start(self, request: JobRequest):
|
|
1274
|
+
with self.lock:
|
|
1275
|
+
if self.active and self.active["status"] in ACTIVE:
|
|
1276
|
+
raise HTTPException(
|
|
1277
|
+
409, "A generation is already running; cancel or finish it first"
|
|
1278
|
+
)
|
|
1279
|
+
self.directory(request.output_location, request.output_folder)
|
|
1280
|
+
self.selected_fonts(request.fonts)
|
|
1281
|
+
if request.source == "custom":
|
|
1282
|
+
capacity = self.capacity(
|
|
1283
|
+
CapacityRequest(
|
|
1284
|
+
text=request.text, language=request.language, languages=request.languages
|
|
1285
|
+
)
|
|
1286
|
+
)
|
|
1287
|
+
if capacity["unsupported_texts"]:
|
|
1288
|
+
raise HTTPException(
|
|
1289
|
+
422,
|
|
1290
|
+
{
|
|
1291
|
+
"message": (
|
|
1292
|
+
"Installed fonts cannot render every custom line; "
|
|
1293
|
+
"edit the text or add compatible fonts"
|
|
1294
|
+
),
|
|
1295
|
+
**capacity,
|
|
1296
|
+
},
|
|
1297
|
+
)
|
|
1298
|
+
if request.samples > capacity["available_pairs"]:
|
|
1299
|
+
raise HTTPException(
|
|
1300
|
+
422,
|
|
1301
|
+
{
|
|
1302
|
+
"message": (
|
|
1303
|
+
f"Your {capacity['texts']} distinct lines support "
|
|
1304
|
+
f"{capacity['available_pairs']} unique text/font pairs. "
|
|
1305
|
+
"Request fewer images or add more text"
|
|
1306
|
+
),
|
|
1307
|
+
**capacity,
|
|
1308
|
+
},
|
|
1309
|
+
)
|
|
1310
|
+
tags = _language_tags(request.language, request.languages)
|
|
1311
|
+
required = (
|
|
1312
|
+
{row["attribution"]["source_id"] for row in spelling_cases() if row["language"] in tags}
|
|
1313
|
+
if request.source == "spelling"
|
|
1314
|
+
else set()
|
|
1315
|
+
)
|
|
1316
|
+
if request.samples < len(required):
|
|
1317
|
+
raise HTTPException(422, f"This spelling check needs at least {len(required)} images")
|
|
1318
|
+
with self.lock:
|
|
1319
|
+
if self.active and self.active["status"] in ACTIVE:
|
|
1320
|
+
raise HTTPException(
|
|
1321
|
+
409, "A generation is already running; finish or cancel it first"
|
|
1322
|
+
)
|
|
1323
|
+
identifier = uuid4().hex
|
|
1324
|
+
name = request.name or f"{request.language}-{identifier[:8]}"
|
|
1325
|
+
job = {
|
|
1326
|
+
"id": identifier,
|
|
1327
|
+
"name": name,
|
|
1328
|
+
"language": request.language,
|
|
1329
|
+
"languages": tags,
|
|
1330
|
+
"source": request.source,
|
|
1331
|
+
"status": "queued",
|
|
1332
|
+
"phase": "queued",
|
|
1333
|
+
"generated": 0,
|
|
1334
|
+
"requested": request.samples,
|
|
1335
|
+
"progress": 0,
|
|
1336
|
+
"output_location": request.output_location,
|
|
1337
|
+
"output_folder": request.output_folder,
|
|
1338
|
+
"output_root": str(self.location_root(request.output_location)),
|
|
1339
|
+
"error": None,
|
|
1340
|
+
"result": None,
|
|
1341
|
+
"created_at": _now(),
|
|
1342
|
+
"updated_at": _now(),
|
|
1343
|
+
"settings": request.model_dump(exclude={"text"}),
|
|
1344
|
+
}
|
|
1345
|
+
destination = self.dataset(job)
|
|
1346
|
+
job["output"] = str(destination)
|
|
1347
|
+
if destination.exists():
|
|
1348
|
+
raise HTTPException(409, "That dataset name already exists; choose a fresh name")
|
|
1349
|
+
if request.source == "upload":
|
|
1350
|
+
with self.upload_lock:
|
|
1351
|
+
upload, path = self.uploaded(request.upload_id)
|
|
1352
|
+
selected = {canonical_language(tag) for tag in tags}
|
|
1353
|
+
upload_selection = upload.get(
|
|
1354
|
+
"selected_languages", _language_tags(upload["language"])
|
|
1355
|
+
)
|
|
1356
|
+
if upload["unlabelled_records"] and set(tags) != set(upload_selection):
|
|
1357
|
+
raise HTTPException(
|
|
1358
|
+
422,
|
|
1359
|
+
"This file contains unlabelled text. Keep its selected language, "
|
|
1360
|
+
"or upload it again after changing the language",
|
|
1361
|
+
)
|
|
1362
|
+
if selected and not selected.intersection(upload["languages"]):
|
|
1363
|
+
raise HTTPException(422, "The uploaded file has no text in this language")
|
|
1364
|
+
if sha256(path) != upload["sha256"]:
|
|
1365
|
+
raise HTTPException(422, "The uploaded file changed; upload it again")
|
|
1366
|
+
self._claimed_uploads.add(request.upload_id)
|
|
1367
|
+
job["upload"] = upload
|
|
1368
|
+
self.active = job
|
|
1369
|
+
self.cancel_event = threading.Event()
|
|
1370
|
+
self._save(job)
|
|
1371
|
+
try:
|
|
1372
|
+
self.executor.submit(self._run, job, request, required, self.cancel_event)
|
|
1373
|
+
except BaseException:
|
|
1374
|
+
self._claimed_uploads.discard(request.upload_id)
|
|
1375
|
+
raise
|
|
1376
|
+
return job.copy()
|
|
1377
|
+
|
|
1378
|
+
def cancel(self, identifier):
|
|
1379
|
+
with self.lock:
|
|
1380
|
+
job = self.job(identifier)
|
|
1381
|
+
if self.active and self.active["id"] == identifier and job["status"] in ACTIVE:
|
|
1382
|
+
self.cancel_event.set()
|
|
1383
|
+
self.active["status"] = "cancelling"
|
|
1384
|
+
self.active["phase"] = "Finishing the current operation before stopping"
|
|
1385
|
+
self._save(self.active)
|
|
1386
|
+
return self.active.copy()
|
|
1387
|
+
return job
|
|
1388
|
+
|
|
1389
|
+
def _run(self, job, request, required, cancellation):
|
|
1390
|
+
last_saved = 0
|
|
1391
|
+
|
|
1392
|
+
def update(progress):
|
|
1393
|
+
nonlocal last_saved
|
|
1394
|
+
with self.lock:
|
|
1395
|
+
job["generated"] = progress.get("generated", job["generated"])
|
|
1396
|
+
job["progress"] = round(100 * job["generated"] / job["requested"], 1)
|
|
1397
|
+
if not cancellation.is_set():
|
|
1398
|
+
job["status"] = progress["stage"]
|
|
1399
|
+
job["phase"] = progress["stage"]
|
|
1400
|
+
if time.monotonic() - last_saved >= 0.5 or progress["stage"] == "verifying":
|
|
1401
|
+
self._save(job)
|
|
1402
|
+
last_saved = time.monotonic()
|
|
1403
|
+
|
|
1404
|
+
def stopped():
|
|
1405
|
+
if cancellation.is_set():
|
|
1406
|
+
raise GenerationCancelled("Generation cancelled; partial output has been retained")
|
|
1407
|
+
|
|
1408
|
+
try:
|
|
1409
|
+
tags = _language_tags(request.language, request.languages)
|
|
1410
|
+
max_chars = self.config.data.max_chars
|
|
1411
|
+
if required:
|
|
1412
|
+
max_chars = max(
|
|
1413
|
+
max_chars,
|
|
1414
|
+
max(len(row["text"]) for row in spelling_cases() if row["language"] in tags),
|
|
1415
|
+
)
|
|
1416
|
+
config = replace(
|
|
1417
|
+
self.config,
|
|
1418
|
+
data=replace(self.config.data, normalization="NFC", max_chars=max_chars),
|
|
1419
|
+
fonts=replace(
|
|
1420
|
+
self.config.fonts,
|
|
1421
|
+
directories=[str(path) for path in self.font_directories()],
|
|
1422
|
+
include=request.fonts,
|
|
1423
|
+
coverage="strict",
|
|
1424
|
+
cache_size=min(self.config.fonts.cache_size, 8),
|
|
1425
|
+
coverage_cache_size=min(self.config.fonts.coverage_cache_size, 128),
|
|
1426
|
+
),
|
|
1427
|
+
generation=replace(
|
|
1428
|
+
self.config.generation,
|
|
1429
|
+
output=str(self.dataset(job)),
|
|
1430
|
+
samples=request.samples,
|
|
1431
|
+
font_size=request.font_size,
|
|
1432
|
+
format=request.format,
|
|
1433
|
+
**{name: getattr(request, name) for name in AUGMENTATION_FIELDS},
|
|
1434
|
+
seed=request.seed,
|
|
1435
|
+
workers=request.workers,
|
|
1436
|
+
train_ratio=request.train_ratio,
|
|
1437
|
+
validation_ratio=request.validation_ratio,
|
|
1438
|
+
languages=tags,
|
|
1439
|
+
language_weights=request.language_weights,
|
|
1440
|
+
language="",
|
|
1441
|
+
direction="auto",
|
|
1442
|
+
suite="kurdish-spelling" if required else "",
|
|
1443
|
+
exclude_genres=[]
|
|
1444
|
+
if required
|
|
1445
|
+
else [*self.config.generation.exclude_genres, "spelling"],
|
|
1446
|
+
),
|
|
1447
|
+
).validate()
|
|
1448
|
+
stopped()
|
|
1449
|
+
with tempfile.TemporaryDirectory(prefix="synthocr-studio-") as directory:
|
|
1450
|
+
custom = None
|
|
1451
|
+
uploaded = None
|
|
1452
|
+
if request.source == "custom":
|
|
1453
|
+
raw, processed = Path(directory) / "raw", Path(directory) / "processed"
|
|
1454
|
+
custom = _custom_source(raw, request.text, tags[0])
|
|
1455
|
+
config = replace(
|
|
1456
|
+
config,
|
|
1457
|
+
data=replace(
|
|
1458
|
+
config.data,
|
|
1459
|
+
raw=str(raw),
|
|
1460
|
+
processed=str(processed),
|
|
1461
|
+
source_languages={},
|
|
1462
|
+
deduplicate=True,
|
|
1463
|
+
write_jsonl=False,
|
|
1464
|
+
),
|
|
1465
|
+
)
|
|
1466
|
+
elif request.source == "upload":
|
|
1467
|
+
upload, source = self.uploaded(request.upload_id)
|
|
1468
|
+
raw, processed = Path(directory) / "raw", Path(directory) / "processed"
|
|
1469
|
+
raw.mkdir()
|
|
1470
|
+
uploaded = raw / source.name
|
|
1471
|
+
shutil.copyfile(source, uploaded)
|
|
1472
|
+
fallback = upload.get("selected_languages", _language_tags(upload["language"]))
|
|
1473
|
+
config = replace(
|
|
1474
|
+
config,
|
|
1475
|
+
data=replace(
|
|
1476
|
+
config.data,
|
|
1477
|
+
raw=str(raw),
|
|
1478
|
+
processed=str(processed),
|
|
1479
|
+
source_languages={uploaded.name: fallback[0]}
|
|
1480
|
+
if len(fallback) == 1
|
|
1481
|
+
else {},
|
|
1482
|
+
deduplicate=True,
|
|
1483
|
+
write_jsonl=False,
|
|
1484
|
+
),
|
|
1485
|
+
)
|
|
1486
|
+
try:
|
|
1487
|
+
result = generate(config, progress=update, cancel=cancellation.is_set)
|
|
1488
|
+
finally:
|
|
1489
|
+
# Retain user raw data alongside partial outputs as well as completed ones.
|
|
1490
|
+
if custom and self.dataset(job).is_dir():
|
|
1491
|
+
shutil.copyfile(custom, self.dataset(job) / "source.jsonl")
|
|
1492
|
+
(self.dataset(job) / "source.txt").write_text(
|
|
1493
|
+
request.text, encoding="utf-8"
|
|
1494
|
+
)
|
|
1495
|
+
if uploaded and self.dataset(job).is_dir():
|
|
1496
|
+
target = self.dataset(job) / uploaded.name
|
|
1497
|
+
shutil.copyfile(uploaded, target)
|
|
1498
|
+
_atomic_json(self.dataset(job) / "source-upload.json", upload)
|
|
1499
|
+
with self.upload_lock:
|
|
1500
|
+
shutil.rmtree(self.uploads / request.upload_id)
|
|
1501
|
+
self._used_uploads[request.upload_id] = job["name"]
|
|
1502
|
+
if len(self._used_uploads) > 20:
|
|
1503
|
+
self._used_uploads.popitem(last=False)
|
|
1504
|
+
stopped()
|
|
1505
|
+
update({"stage": "verifying"})
|
|
1506
|
+
result["verification"] = verify(self.dataset(job))
|
|
1507
|
+
if required:
|
|
1508
|
+
missing = required.copy()
|
|
1509
|
+
with (self.dataset(job) / "manifest.jsonl").open(encoding="utf-8") as stream:
|
|
1510
|
+
for record in map(json.loads, stream):
|
|
1511
|
+
missing.discard(record["attribution"]["source_id"])
|
|
1512
|
+
if missing:
|
|
1513
|
+
raise ValueError(f"Spelling check missed {len(missing)} cases; inspect fonts")
|
|
1514
|
+
result["spelling_cases_checked"] = len(required)
|
|
1515
|
+
stopped()
|
|
1516
|
+
with self.lock:
|
|
1517
|
+
job.update(status="completed", phase="verified", result=result, progress=100)
|
|
1518
|
+
except GenerationCancelled as error:
|
|
1519
|
+
with self.lock:
|
|
1520
|
+
job.update(status="cancelled", phase="cancelled", error=str(error))
|
|
1521
|
+
except Exception as error:
|
|
1522
|
+
with self.lock:
|
|
1523
|
+
job.update(status="failed", phase="failed", error=str(error))
|
|
1524
|
+
finally:
|
|
1525
|
+
with self.upload_lock:
|
|
1526
|
+
self._claimed_uploads.discard(request.upload_id)
|
|
1527
|
+
with self.lock:
|
|
1528
|
+
self._save(job)
|
|
1529
|
+
|
|
1530
|
+
def close(self):
|
|
1531
|
+
self.cancel_event.set()
|
|
1532
|
+
try:
|
|
1533
|
+
self.executor.shutdown(wait=True, cancel_futures=True)
|
|
1534
|
+
finally:
|
|
1535
|
+
for lock in reversed(self._root_locks):
|
|
1536
|
+
lock.close()
|
|
1537
|
+
|
|
1538
|
+
def capacity(self, request: CapacityRequest):
|
|
1539
|
+
with self.preview_lock:
|
|
1540
|
+
key = hashlib.sha256(
|
|
1541
|
+
(
|
|
1542
|
+
"\0".join(_language_tags(request.language, request.languages))
|
|
1543
|
+
+ "\0"
|
|
1544
|
+
+ request.text
|
|
1545
|
+
).encode("utf-8")
|
|
1546
|
+
).hexdigest()
|
|
1547
|
+
if key in self._capacities:
|
|
1548
|
+
self._capacities.move_to_end(key)
|
|
1549
|
+
return self._capacities[key]
|
|
1550
|
+
result = self._capacity(request)
|
|
1551
|
+
self._capacities[key] = result
|
|
1552
|
+
if len(self._capacities) > 4:
|
|
1553
|
+
self._capacities.popitem(last=False)
|
|
1554
|
+
return result
|
|
1555
|
+
|
|
1556
|
+
def _capacity(self, request: CapacityRequest):
|
|
1557
|
+
result = {
|
|
1558
|
+
"texts": 0,
|
|
1559
|
+
"available_pairs": 0,
|
|
1560
|
+
"unsupported_texts": 0,
|
|
1561
|
+
"unsupported_examples": [],
|
|
1562
|
+
}
|
|
1563
|
+
if not required_codepoints(request.text):
|
|
1564
|
+
return result
|
|
1565
|
+
with tempfile.TemporaryDirectory(prefix="synthocr-capacity-") as directory:
|
|
1566
|
+
raw, processed = Path(directory) / "raw", Path(directory) / "processed"
|
|
1567
|
+
_custom_source(
|
|
1568
|
+
raw, request.text, _language_tags(request.language, request.languages)[0]
|
|
1569
|
+
)
|
|
1570
|
+
config = replace(
|
|
1571
|
+
self.config,
|
|
1572
|
+
data=replace(
|
|
1573
|
+
self.config.data,
|
|
1574
|
+
raw=str(raw),
|
|
1575
|
+
processed=str(processed),
|
|
1576
|
+
source_languages={},
|
|
1577
|
+
normalization="NFC",
|
|
1578
|
+
deduplicate=True,
|
|
1579
|
+
),
|
|
1580
|
+
)
|
|
1581
|
+
prepare(config)
|
|
1582
|
+
coverage = self._coverage
|
|
1583
|
+
fonts = self.fonts()
|
|
1584
|
+
with sqlite3.connect(
|
|
1585
|
+
(processed / "texts.sqlite3").as_uri() + "?mode=ro", uri=True
|
|
1586
|
+
) as db:
|
|
1587
|
+
db.execute("PRAGMA cache_size=-4096")
|
|
1588
|
+
for (text,) in db.execute("SELECT text FROM texts ORDER BY id"):
|
|
1589
|
+
needed = required_codepoints(text)
|
|
1590
|
+
compatible = sum(not coverage.missing(font, needed) for font in fonts)
|
|
1591
|
+
result["texts"] += 1
|
|
1592
|
+
result["available_pairs"] += compatible
|
|
1593
|
+
if compatible == 0:
|
|
1594
|
+
result["unsupported_texts"] += 1
|
|
1595
|
+
if len(result["unsupported_examples"]) < 3:
|
|
1596
|
+
result["unsupported_examples"].append(text[:120])
|
|
1597
|
+
return result
|
|
1598
|
+
|
|
1599
|
+
def preview(self, request: PreviewRequest):
|
|
1600
|
+
# Keep native font allocations bounded even when the browser types quickly.
|
|
1601
|
+
with self.preview_lock:
|
|
1602
|
+
return self._preview(request)
|
|
1603
|
+
|
|
1604
|
+
def _corpus_preview(self, request):
|
|
1605
|
+
"""Read one indexed label; never prepare or count a corpus in an HTTP request."""
|
|
1606
|
+
tags = _language_tags(request.language, request.languages)
|
|
1607
|
+
excluded = sorted({*self.config.generation.exclude_genres, "spelling"})
|
|
1608
|
+
path = self.config.path(self.config.data.processed) / "texts.sqlite3"
|
|
1609
|
+
message = "Prepare the corpus with synthocr prepare, or upload a text file"
|
|
1610
|
+
if not path.is_file():
|
|
1611
|
+
raise HTTPException(400, message)
|
|
1612
|
+
db = None
|
|
1613
|
+
try:
|
|
1614
|
+
db = sqlite3.connect(path.as_uri() + "?mode=ro", uri=True)
|
|
1615
|
+
db.execute("PRAGMA cache_size=-1024")
|
|
1616
|
+
db.execute("PRAGMA temp_store=FILE")
|
|
1617
|
+
|
|
1618
|
+
def one(selected, offset, representative=False):
|
|
1619
|
+
conditions, parameters = [], []
|
|
1620
|
+
if selected:
|
|
1621
|
+
conditions.append(f"language IN ({','.join('?' for _ in selected)})")
|
|
1622
|
+
parameters.extend(selected)
|
|
1623
|
+
conditions.append(f"genre NOT IN ({','.join('?' for _ in excluded)})")
|
|
1624
|
+
parameters.extend(excluded)
|
|
1625
|
+
if representative:
|
|
1626
|
+
conditions.append("genre IN ('prose','dialogue') AND length(text)>=12")
|
|
1627
|
+
parameters.append(offset)
|
|
1628
|
+
# Use the language index when available and stop after at most
|
|
1629
|
+
# 32 eligible labels. Avoid sorting corpora prepared without deduplication.
|
|
1630
|
+
return db.execute(
|
|
1631
|
+
"SELECT text,language FROM texts WHERE "
|
|
1632
|
+
+ " AND ".join(conditions)
|
|
1633
|
+
+ " LIMIT 1 OFFSET ?",
|
|
1634
|
+
parameters,
|
|
1635
|
+
).fetchone()
|
|
1636
|
+
|
|
1637
|
+
selected = [tags[request.index % len(tags)]] if len(tags) > 1 else tags
|
|
1638
|
+
offset = request.index // len(tags) if len(tags) > 1 else request.index
|
|
1639
|
+
# Prefer readable natural text for the preview. Short labels, numbers
|
|
1640
|
+
# and punctuation remain available to generation without restriction.
|
|
1641
|
+
row = (
|
|
1642
|
+
one(selected, offset, True)
|
|
1643
|
+
or one(selected, 0, True)
|
|
1644
|
+
or one(selected, offset)
|
|
1645
|
+
or one(selected, 0)
|
|
1646
|
+
)
|
|
1647
|
+
if row is None and len(tags) > 1:
|
|
1648
|
+
row = one(tags, 0)
|
|
1649
|
+
if row is None:
|
|
1650
|
+
raise HTTPException(400, "No usable corpus text for this language. " + message)
|
|
1651
|
+
return row[0], canonical_language(row[1])
|
|
1652
|
+
except sqlite3.Error as error:
|
|
1653
|
+
raise HTTPException(400, message) from error
|
|
1654
|
+
finally:
|
|
1655
|
+
if db is not None:
|
|
1656
|
+
db.close()
|
|
1657
|
+
|
|
1658
|
+
def _spelling_preview(self, request):
|
|
1659
|
+
tags = _language_tags(request.language, request.languages)
|
|
1660
|
+
if not tags or any(tag not in {"ckb", "kmr"} for tag in tags):
|
|
1661
|
+
raise HTTPException(422, "Spelling examples support Sorani, Kurmanji or both")
|
|
1662
|
+
tag = tags[request.index % len(tags)]
|
|
1663
|
+
# This is the fixed, bundled spelling catalog, rather than a user corpus.
|
|
1664
|
+
cases = list(spelling_cases(tag))
|
|
1665
|
+
case = cases[(request.index // len(tags)) % len(cases)]
|
|
1666
|
+
return case["text"], case["language"], case["attribution"]["source_id"]
|
|
1667
|
+
|
|
1668
|
+
def _preview(self, request: PreviewRequest):
|
|
1669
|
+
if request.language not in LANGUAGES and not LANGUAGE.fullmatch(request.language):
|
|
1670
|
+
raise HTTPException(422, "Choose a valid language")
|
|
1671
|
+
default = {
|
|
1672
|
+
"sorani": "سڵاو جیهان",
|
|
1673
|
+
"ckb": "سڵاو جیهان",
|
|
1674
|
+
"kurmanji": "Silav dinya",
|
|
1675
|
+
"kmr": "Silav dinya",
|
|
1676
|
+
"both": "سڵاو · Silav",
|
|
1677
|
+
"all": "سڵاو Hello 123",
|
|
1678
|
+
"ar": "مرحبا بالعالم",
|
|
1679
|
+
"fa": "سلام دنیا",
|
|
1680
|
+
"en": "Hello world",
|
|
1681
|
+
"he": "שלום עולם",
|
|
1682
|
+
"hi": "नमस्ते दुनिया",
|
|
1683
|
+
"zh": "你好世界",
|
|
1684
|
+
}
|
|
1685
|
+
tags = _language_tags(request.language, request.languages)
|
|
1686
|
+
source_language = tags[0] if len(tags) == 1 else ""
|
|
1687
|
+
case_id = None
|
|
1688
|
+
if request.source == "corpus":
|
|
1689
|
+
original, source_language = self._corpus_preview(request)
|
|
1690
|
+
elif request.source == "spelling":
|
|
1691
|
+
original, source_language, case_id = self._spelling_preview(request)
|
|
1692
|
+
else:
|
|
1693
|
+
if request.languages is not None and not request.text:
|
|
1694
|
+
source_language = tags[request.index % len(tags)]
|
|
1695
|
+
original = request.text or default.get(
|
|
1696
|
+
source_language, default.get(request.language, "123")
|
|
1697
|
+
)
|
|
1698
|
+
first_line = next((line for line in io.StringIO(original) if required_codepoints(line)), "")
|
|
1699
|
+
text = next(chunks(normalize(first_line), self.config.data.max_chars), "")
|
|
1700
|
+
if not required_codepoints(text):
|
|
1701
|
+
raise HTTPException(422, "Enter visible text or numbers")
|
|
1702
|
+
fonts = self.selected_fonts(request.fonts)
|
|
1703
|
+
if request.fonts:
|
|
1704
|
+
# Show the user's chosen fonts in turn as they ask for more examples.
|
|
1705
|
+
offset = request.index % len(fonts)
|
|
1706
|
+
else:
|
|
1707
|
+
# Start with a readable Kurdish face while still using shared coverage selection.
|
|
1708
|
+
preferred = "NotoNaskhArabic" if source_language in {"ckb", "ar"} else "NotoSans["
|
|
1709
|
+
offset = next((i for i, face in enumerate(fonts) if preferred in face.name), 0)
|
|
1710
|
+
face = self._coverage.choose(fonts, text, offset)
|
|
1711
|
+
if face is None:
|
|
1712
|
+
raise HTTPException(422, "No installed font supports this complete line")
|
|
1713
|
+
settings = replace(
|
|
1714
|
+
self.config.generation,
|
|
1715
|
+
font_size=request.font_size,
|
|
1716
|
+
direction="auto",
|
|
1717
|
+
language="",
|
|
1718
|
+
**{name: getattr(request, name) for name in AUGMENTATION_FIELDS},
|
|
1719
|
+
max_width=2048,
|
|
1720
|
+
max_height=256,
|
|
1721
|
+
max_pixels=524_288,
|
|
1722
|
+
)
|
|
1723
|
+
|
|
1724
|
+
def render(appearance):
|
|
1725
|
+
image, info = Renderer(appearance, cache_size=1).render(
|
|
1726
|
+
text, face, request.seed, source_language
|
|
1727
|
+
)
|
|
1728
|
+
try:
|
|
1729
|
+
with io.BytesIO() as buffer:
|
|
1730
|
+
image.save(buffer, format="PNG")
|
|
1731
|
+
encoded = base64.b64encode(buffer.getvalue()).decode("ascii")
|
|
1732
|
+
finally:
|
|
1733
|
+
image.close()
|
|
1734
|
+
return {"image": f"data:image/png;base64,{encoded}", **info}
|
|
1735
|
+
|
|
1736
|
+
# Hold the sample, font and seed fixed while comparing effects. Only one
|
|
1737
|
+
# decoded image is alive at once, bounded by the preview's pixel limit.
|
|
1738
|
+
before = render(replace(settings, augmentation=0)) if not request.augmentation else None
|
|
1739
|
+
if request.compare:
|
|
1740
|
+
# Hidden recipe selectors retain their existing thumbnails while
|
|
1741
|
+
# disabled. Do not serialize a catalog of identical clean images.
|
|
1742
|
+
presets = list(DOCUMENT_PRESETS) if request.augmentation else []
|
|
1743
|
+
else:
|
|
1744
|
+
presets = request.background_presets
|
|
1745
|
+
variants = []
|
|
1746
|
+
distribution = background_distribution(settings)
|
|
1747
|
+
for preset in presets:
|
|
1748
|
+
rendered = (
|
|
1749
|
+
render(replace(settings, background_presets=[preset], background_weights={}))
|
|
1750
|
+
if request.augmentation
|
|
1751
|
+
else before
|
|
1752
|
+
)
|
|
1753
|
+
if rendered["augmentation"]:
|
|
1754
|
+
rendered["augmentation"]["background_probability"] = distribution.get(preset, 0)
|
|
1755
|
+
variants.append(
|
|
1756
|
+
{
|
|
1757
|
+
"preset": preset,
|
|
1758
|
+
"name": DOCUMENT_PRESETS[preset]["label"],
|
|
1759
|
+
"text": text,
|
|
1760
|
+
"font": face.name,
|
|
1761
|
+
**rendered,
|
|
1762
|
+
}
|
|
1763
|
+
)
|
|
1764
|
+
if variants:
|
|
1765
|
+
selected = next((name for name, share in distribution.items() if share > 0), presets[0])
|
|
1766
|
+
after = next(variant for variant in variants if variant["preset"] == selected)
|
|
1767
|
+
else:
|
|
1768
|
+
after = render(settings) if request.augmentation else before
|
|
1769
|
+
if not request.compare or request.augmentation:
|
|
1770
|
+
variants.append(
|
|
1771
|
+
{
|
|
1772
|
+
"preset": "",
|
|
1773
|
+
"name": "Custom background",
|
|
1774
|
+
"text": text,
|
|
1775
|
+
"font": face.name,
|
|
1776
|
+
**after,
|
|
1777
|
+
}
|
|
1778
|
+
)
|
|
1779
|
+
if before is None:
|
|
1780
|
+
# Bounds can shrink a long rotated line. Compare it with clean ink
|
|
1781
|
+
# at that same effective point size rather than a larger baseline.
|
|
1782
|
+
font_size = min(variant["font_size"] for variant in variants)
|
|
1783
|
+
before = render(replace(settings, augmentation=0, font_size=font_size))
|
|
1784
|
+
return {
|
|
1785
|
+
**after,
|
|
1786
|
+
"text": text,
|
|
1787
|
+
"font": face.name,
|
|
1788
|
+
"wrapped": text != normalize(original),
|
|
1789
|
+
"source": request.source,
|
|
1790
|
+
"source_language": source_language,
|
|
1791
|
+
"case_id": case_id,
|
|
1792
|
+
"seed": request.seed,
|
|
1793
|
+
"before": {"text": text, "font": face.name, **before},
|
|
1794
|
+
"variants": variants,
|
|
1795
|
+
}
|
|
1796
|
+
|
|
1797
|
+
def _sample_offset(self, job, manifest, offset):
|
|
1798
|
+
if job["status"] != "completed" or not offset:
|
|
1799
|
+
return 0, 0
|
|
1800
|
+
info = manifest.stat()
|
|
1801
|
+
fingerprint = hashlib.sha256(
|
|
1802
|
+
f"{info.st_dev}:{info.st_ino}:{info.st_size}:{info.st_mtime_ns}".encode()
|
|
1803
|
+
).hexdigest()[:16]
|
|
1804
|
+
index = self.cache / f"{job['id']}-{fingerprint}.index"
|
|
1805
|
+
with self.archive_lock:
|
|
1806
|
+
if index.is_symlink():
|
|
1807
|
+
raise HTTPException(400, "Invalid gallery index")
|
|
1808
|
+
if not index.exists():
|
|
1809
|
+
# Eight bytes per 128 records, rather than an in-memory row index.
|
|
1810
|
+
self._trim_cache(reserve=(100_000 // INDEX_STRIDE + 1) * 8)
|
|
1811
|
+
temporary = self.cache / f"{uuid4().hex}.tmp"
|
|
1812
|
+
try:
|
|
1813
|
+
with manifest.open("rb") as stream, temporary.open("xb") as target:
|
|
1814
|
+
line = 0
|
|
1815
|
+
while True:
|
|
1816
|
+
position = stream.tell()
|
|
1817
|
+
if not stream.readline():
|
|
1818
|
+
break
|
|
1819
|
+
if line >= 100_000:
|
|
1820
|
+
raise HTTPException(400, "Dataset exceeds the studio sample limit")
|
|
1821
|
+
if line % INDEX_STRIDE == 0:
|
|
1822
|
+
target.write(struct.pack("<Q", position))
|
|
1823
|
+
line += 1
|
|
1824
|
+
os.replace(temporary, index)
|
|
1825
|
+
finally:
|
|
1826
|
+
temporary.unlink(missing_ok=True)
|
|
1827
|
+
index.touch()
|
|
1828
|
+
entries = index.stat().st_size // 8
|
|
1829
|
+
if not entries:
|
|
1830
|
+
return 0, 0
|
|
1831
|
+
block = min(offset // INDEX_STRIDE, entries - 1)
|
|
1832
|
+
with index.open("rb") as stream:
|
|
1833
|
+
stream.seek(block * 8)
|
|
1834
|
+
return struct.unpack("<Q", stream.read(8))[0], block * INDEX_STRIDE
|
|
1835
|
+
|
|
1836
|
+
def samples(self, job, offset, limit):
|
|
1837
|
+
path = self.dataset(job) / "manifest.jsonl"
|
|
1838
|
+
rows = []
|
|
1839
|
+
if path.is_file() and not path.is_symlink():
|
|
1840
|
+
position, first = self._sample_offset(job, path, offset)
|
|
1841
|
+
with path.open("rb") as stream:
|
|
1842
|
+
stream.seek(position)
|
|
1843
|
+
for index, line in enumerate(stream, first):
|
|
1844
|
+
if index < offset:
|
|
1845
|
+
continue
|
|
1846
|
+
if len(rows) >= limit:
|
|
1847
|
+
break
|
|
1848
|
+
try:
|
|
1849
|
+
row = json.loads(line)
|
|
1850
|
+
except ValueError as error:
|
|
1851
|
+
if job["status"] == "completed":
|
|
1852
|
+
raise HTTPException(400, "Invalid sample in dataset") from error
|
|
1853
|
+
break # A running worker may still be flushing its final line.
|
|
1854
|
+
if not isinstance(row, dict) or not isinstance(row.get("image"), str):
|
|
1855
|
+
raise HTTPException(400, "Invalid sample in dataset")
|
|
1856
|
+
filename = Path(row["image"]).name
|
|
1857
|
+
if not IMAGE.fullmatch(filename):
|
|
1858
|
+
raise HTTPException(400, "Invalid image name in dataset")
|
|
1859
|
+
rows.append(
|
|
1860
|
+
{
|
|
1861
|
+
key: row.get(key)
|
|
1862
|
+
for key in ("id", "text", "font", "language", "split", "augmentation")
|
|
1863
|
+
}
|
|
1864
|
+
| {
|
|
1865
|
+
"image": f"/api/jobs/{job['id']}/images/{filename}",
|
|
1866
|
+
"thumbnail": f"/api/jobs/{job['id']}/thumbnails/{filename}",
|
|
1867
|
+
}
|
|
1868
|
+
)
|
|
1869
|
+
return {"samples": rows, "total": job["generated"], "offset": offset, "limit": limit}
|
|
1870
|
+
|
|
1871
|
+
def image_path(self, job, filename):
|
|
1872
|
+
if not IMAGE.fullmatch(filename):
|
|
1873
|
+
raise HTTPException(404, "Image not found")
|
|
1874
|
+
dataset = self.dataset(job)
|
|
1875
|
+
directory = dataset / "images"
|
|
1876
|
+
path = directory / filename
|
|
1877
|
+
if (
|
|
1878
|
+
directory.is_symlink()
|
|
1879
|
+
or path.is_symlink()
|
|
1880
|
+
or not path.is_file()
|
|
1881
|
+
or path.resolve().parent != directory
|
|
1882
|
+
):
|
|
1883
|
+
raise HTTPException(404, "Image not found")
|
|
1884
|
+
return path
|
|
1885
|
+
|
|
1886
|
+
def thumbnail(self, job, filename):
|
|
1887
|
+
path = self.image_path(job, filename)
|
|
1888
|
+
with self.preview_lock, Image.open(path) as image:
|
|
1889
|
+
if image.width * image.height > 16_777_216:
|
|
1890
|
+
raise HTTPException(400, "Image exceeds the supported preview size")
|
|
1891
|
+
image.thumbnail((768, 192), Image.Resampling.LANCZOS, reducing_gap=3)
|
|
1892
|
+
with io.BytesIO() as buffer:
|
|
1893
|
+
image.save(buffer, format="PNG")
|
|
1894
|
+
return Response(
|
|
1895
|
+
buffer.getvalue(),
|
|
1896
|
+
media_type="image/png",
|
|
1897
|
+
headers={"Cache-Control": "private, max-age=3600"},
|
|
1898
|
+
)
|
|
1899
|
+
|
|
1900
|
+
def download(self, job, format, *, prepare_only=False):
|
|
1901
|
+
if job["status"] != "completed":
|
|
1902
|
+
raise HTTPException(409, "Downloads are available after generation and verification")
|
|
1903
|
+
dataset = self.dataset(job)
|
|
1904
|
+
with self.archive_lock:
|
|
1905
|
+
verify(dataset)
|
|
1906
|
+
prepared = {
|
|
1907
|
+
"url": f"/api/jobs/{job['id']}/download?format={format}",
|
|
1908
|
+
"filename": f"{job['name']}-{format}.zip"
|
|
1909
|
+
if format != "manifest"
|
|
1910
|
+
else f"{job['name']}-manifest.jsonl",
|
|
1911
|
+
}
|
|
1912
|
+
if format == "manifest":
|
|
1913
|
+
if prepare_only:
|
|
1914
|
+
return prepared
|
|
1915
|
+
return FileResponse(
|
|
1916
|
+
dataset / "manifest.jsonl",
|
|
1917
|
+
media_type="application/x-ndjson",
|
|
1918
|
+
filename=f"{job['name']}-manifest.jsonl",
|
|
1919
|
+
)
|
|
1920
|
+
digests = [
|
|
1921
|
+
sha256(dataset / name) for name in ("manifest.jsonl", "run.json", "stats.json")
|
|
1922
|
+
]
|
|
1923
|
+
for name in ("source.txt", "source.jsonl", "source.csv", "source-upload.json"):
|
|
1924
|
+
if (dataset / name).is_file():
|
|
1925
|
+
digests.append(sha256(dataset / name))
|
|
1926
|
+
digest = hashlib.sha256("".join(digests).encode()).hexdigest()[:16]
|
|
1927
|
+
archive = self.cache / f"{job['id']}-{digest}-{format}.zip"
|
|
1928
|
+
if archive.is_symlink():
|
|
1929
|
+
raise HTTPException(400, "Invalid download archive")
|
|
1930
|
+
if not archive.exists():
|
|
1931
|
+
# The maintained exporter hardlinks images on this filesystem.
|
|
1932
|
+
# Remove its attributed view after archiving instead of keeping
|
|
1933
|
+
# four permanent copies of each generated dataset.
|
|
1934
|
+
with tempfile.TemporaryDirectory(prefix=".export-", dir=self.metadata) as directory:
|
|
1935
|
+
source = dataset
|
|
1936
|
+
if format != "dataset":
|
|
1937
|
+
retained_sources = tuple(
|
|
1938
|
+
dataset / name
|
|
1939
|
+
for name in (
|
|
1940
|
+
"source.txt",
|
|
1941
|
+
"source.jsonl",
|
|
1942
|
+
"source.csv",
|
|
1943
|
+
"source-upload.json",
|
|
1944
|
+
)
|
|
1945
|
+
if (dataset / name).is_file()
|
|
1946
|
+
)
|
|
1947
|
+
retained_bytes = sum(path.stat().st_size for path in retained_sources)
|
|
1948
|
+
# Retained raw inputs are part of every attributed export.
|
|
1949
|
+
# Account for them before creating or copying its view.
|
|
1950
|
+
self._trim_cache(reserve=retained_bytes)
|
|
1951
|
+
source = Path(directory) / "view"
|
|
1952
|
+
try:
|
|
1953
|
+
export(
|
|
1954
|
+
dataset,
|
|
1955
|
+
source,
|
|
1956
|
+
format,
|
|
1957
|
+
max_bytes=self.cache_limit - retained_bytes,
|
|
1958
|
+
)
|
|
1959
|
+
except ValueError as error:
|
|
1960
|
+
if str(error).startswith("Export exceeds the configured byte limit"):
|
|
1961
|
+
raise HTTPException(
|
|
1962
|
+
413,
|
|
1963
|
+
"This export exceeds the studio cache limit. Download the "
|
|
1964
|
+
"manifest, use the dataset in the output folder, or restart "
|
|
1965
|
+
"with a larger --cache-mb.",
|
|
1966
|
+
) from error
|
|
1967
|
+
raise
|
|
1968
|
+
for path in retained_sources:
|
|
1969
|
+
shutil.copyfile(path, source / path.name)
|
|
1970
|
+
self._trim_cache(reserve=_archive_size(source))
|
|
1971
|
+
temporary = self.cache / f"{uuid4().hex}.tmp"
|
|
1972
|
+
try:
|
|
1973
|
+
_stored_zip(source, temporary)
|
|
1974
|
+
os.replace(temporary, archive)
|
|
1975
|
+
finally:
|
|
1976
|
+
temporary.unlink(missing_ok=True)
|
|
1977
|
+
archive.touch()
|
|
1978
|
+
if prepare_only:
|
|
1979
|
+
return prepared
|
|
1980
|
+
self._leased[archive] = self._leased.get(archive, 0) + 1
|
|
1981
|
+
return _ArchiveResponse(
|
|
1982
|
+
archive,
|
|
1983
|
+
media_type="application/zip",
|
|
1984
|
+
filename=f"{job['name']}-{format}.zip",
|
|
1985
|
+
release=self._release_archive,
|
|
1986
|
+
)
|
|
1987
|
+
|
|
1988
|
+
|
|
1989
|
+
def create_app(
|
|
1990
|
+
config: Config,
|
|
1991
|
+
output_root: Path | None = None,
|
|
1992
|
+
*,
|
|
1993
|
+
cache_mb: int = 256,
|
|
1994
|
+
output_locations: dict[str, Path] | None = None,
|
|
1995
|
+
) -> FastAPI:
|
|
1996
|
+
studio = Studio(config, output_root, cache_mb=cache_mb, output_locations=output_locations)
|
|
1997
|
+
|
|
1998
|
+
@asynccontextmanager
|
|
1999
|
+
async def lifespan(app):
|
|
2000
|
+
try:
|
|
2001
|
+
yield
|
|
2002
|
+
finally:
|
|
2003
|
+
studio.close()
|
|
2004
|
+
|
|
2005
|
+
app = FastAPI(title="SynthOCR Studio", version=__version__, lifespan=lifespan)
|
|
2006
|
+
app.state.studio = studio
|
|
2007
|
+
app.add_middleware(TrustedHostMiddleware, allowed_hosts=["localhost", "127.0.0.1", "[::1]"])
|
|
2008
|
+
|
|
2009
|
+
@app.middleware("http")
|
|
2010
|
+
async def local_requests(request: Request, call_next):
|
|
2011
|
+
if request.method not in {"GET", "HEAD", "OPTIONS"}:
|
|
2012
|
+
origin = request.headers.get("origin")
|
|
2013
|
+
if origin:
|
|
2014
|
+
try:
|
|
2015
|
+
parsed = urlsplit(origin)
|
|
2016
|
+
target = request.url
|
|
2017
|
+
same_origin = (
|
|
2018
|
+
parsed.scheme == target.scheme
|
|
2019
|
+
and parsed.username is None
|
|
2020
|
+
and parsed.password is None
|
|
2021
|
+
and not parsed.query
|
|
2022
|
+
and not parsed.fragment
|
|
2023
|
+
and parsed.path in {"", "/"}
|
|
2024
|
+
and parsed.hostname == target.hostname
|
|
2025
|
+
and (parsed.port or (443 if parsed.scheme == "https" else 80))
|
|
2026
|
+
== (target.port or (443 if target.scheme == "https" else 80))
|
|
2027
|
+
)
|
|
2028
|
+
except ValueError:
|
|
2029
|
+
same_origin = False
|
|
2030
|
+
if not same_origin:
|
|
2031
|
+
return JSONResponse({"detail": "Use the studio's own local browser page"}, 403)
|
|
2032
|
+
if request.headers.get("sec-fetch-site") == "cross-site":
|
|
2033
|
+
return JSONResponse({"detail": "Cross-site requests are not allowed"}, 403)
|
|
2034
|
+
length = request.headers.get("content-length")
|
|
2035
|
+
upload = request.method == "POST" and request.url.path == "/api/uploads"
|
|
2036
|
+
remove_upload = request.method == "DELETE" and re.fullmatch(
|
|
2037
|
+
r"/api/(?:uploads/[a-f0-9]{32}|fonts/[a-f0-9]{64})", request.url.path
|
|
2038
|
+
)
|
|
2039
|
+
if request.method == "POST" and request.url.path == "/api/fonts":
|
|
2040
|
+
if length is not None and (not length.isdigit() or int(length) > FONT_UPLOAD_BYTES):
|
|
2041
|
+
return JSONResponse({"detail": "Font files must be at most 32 MiB"}, 413)
|
|
2042
|
+
elif upload:
|
|
2043
|
+
if length is not None and (not length.isdigit() or int(length) > UPLOAD_BYTES):
|
|
2044
|
+
return JSONResponse({"detail": "Uploaded files must be at most 50 MiB"}, 413)
|
|
2045
|
+
elif remove_upload:
|
|
2046
|
+
if length not in {None, "0"}:
|
|
2047
|
+
return JSONResponse({"detail": "Removal does not accept a body"}, 413)
|
|
2048
|
+
elif length is None or not length.isdigit() or int(length) > 256 * 1024:
|
|
2049
|
+
return JSONResponse({"detail": "Request body must be at most 256 KiB"}, 413)
|
|
2050
|
+
response = await call_next(request)
|
|
2051
|
+
response.headers["X-Content-Type-Options"] = "nosniff"
|
|
2052
|
+
response.headers["Referrer-Policy"] = "same-origin"
|
|
2053
|
+
response.headers.setdefault("Cache-Control", "no-store")
|
|
2054
|
+
return response
|
|
2055
|
+
|
|
2056
|
+
@app.exception_handler(ValueError)
|
|
2057
|
+
async def invalid_operation(request, error):
|
|
2058
|
+
return JSONResponse({"detail": str(error)}, 400)
|
|
2059
|
+
|
|
2060
|
+
@app.exception_handler(OSError)
|
|
2061
|
+
async def inaccessible_output(request, error):
|
|
2062
|
+
return JSONResponse({"detail": f"Cannot access output data: {error}"}, 400)
|
|
2063
|
+
|
|
2064
|
+
@app.get("/api/info")
|
|
2065
|
+
def info():
|
|
2066
|
+
corpora = []
|
|
2067
|
+
languages = [
|
|
2068
|
+
{"id": key, "name": name}
|
|
2069
|
+
for key, name in (
|
|
2070
|
+
("sorani", "Sorani · سۆرانی"),
|
|
2071
|
+
("kurmanji", "Kurmanji · Kurmancî"),
|
|
2072
|
+
("both", "Both Kurdish varieties"),
|
|
2073
|
+
("all", "All source languages"),
|
|
2074
|
+
)
|
|
2075
|
+
]
|
|
2076
|
+
registry = config.path(config.data.processed) / "corpora.json"
|
|
2077
|
+
if registry.is_file() and registry.stat().st_size < 1024 * 1024:
|
|
2078
|
+
try:
|
|
2079
|
+
for corpus in json.loads(registry.read_text(encoding="utf-8"))["corpora"]:
|
|
2080
|
+
tag, records = corpus["language"], corpus["records"]
|
|
2081
|
+
if not isinstance(tag, str) or type(records) is not int or records < 1:
|
|
2082
|
+
continue
|
|
2083
|
+
if not tag or LANGUAGE.fullmatch(tag):
|
|
2084
|
+
corpora.append({"language": tag, "records": records})
|
|
2085
|
+
if tag not in {"ckb", "kmr", ""}:
|
|
2086
|
+
languages.append({"id": tag, "name": tag, "records": records})
|
|
2087
|
+
except OSError, ValueError, TypeError, KeyError:
|
|
2088
|
+
pass
|
|
2089
|
+
return {
|
|
2090
|
+
"version": __version__,
|
|
2091
|
+
"output_root": str(studio.output_root),
|
|
2092
|
+
"output_locations": [
|
|
2093
|
+
{
|
|
2094
|
+
"id": name,
|
|
2095
|
+
"label": "Dataset storage"
|
|
2096
|
+
if name == "default"
|
|
2097
|
+
else name.replace("-", " ").capitalize(),
|
|
2098
|
+
}
|
|
2099
|
+
for name in studio.output_locations
|
|
2100
|
+
],
|
|
2101
|
+
"fonts_count": len(studio.fonts()),
|
|
2102
|
+
"raqm": features.check_feature("raqm"),
|
|
2103
|
+
"languages": languages,
|
|
2104
|
+
"corpora": corpora,
|
|
2105
|
+
"storage": studio.storage(),
|
|
2106
|
+
"upload_limit_bytes": UPLOAD_BYTES,
|
|
2107
|
+
"upload_formats": ["txt", "jsonl", "csv"],
|
|
2108
|
+
"document_presets": DOCUMENT_PRESETS,
|
|
2109
|
+
"document_layers": [
|
|
2110
|
+
{"id": name, "label": label, "description": description}
|
|
2111
|
+
for name, (label, description) in DOCUMENT_LAYER_LABELS.items()
|
|
2112
|
+
],
|
|
2113
|
+
"image_presets": IMAGE_PRESETS,
|
|
2114
|
+
"recommended_preset": "balanced",
|
|
2115
|
+
"defaults": JobRequest().model_dump(exclude={"name", "source", "text", "upload_id"}),
|
|
2116
|
+
}
|
|
2117
|
+
|
|
2118
|
+
@app.get("/api/directories")
|
|
2119
|
+
def directories(
|
|
2120
|
+
location: str = Query("default", max_length=32),
|
|
2121
|
+
path: str = Query("", max_length=512),
|
|
2122
|
+
):
|
|
2123
|
+
return studio.directories(location, path)
|
|
2124
|
+
|
|
2125
|
+
@app.post("/api/directories", status_code=201)
|
|
2126
|
+
def create_directory(request: DirectoryRequest):
|
|
2127
|
+
return studio.create_directory(request)
|
|
2128
|
+
|
|
2129
|
+
@app.get("/api/fonts")
|
|
2130
|
+
def list_fonts():
|
|
2131
|
+
return {"fonts": studio.font_catalog()[0]}
|
|
2132
|
+
|
|
2133
|
+
@app.post("/api/fonts", status_code=201)
|
|
2134
|
+
async def add_font(request: Request, filename: str = Query(..., min_length=1, max_length=255)):
|
|
2135
|
+
return await studio.add_font(request, filename)
|
|
2136
|
+
|
|
2137
|
+
@app.delete("/api/fonts/{digest}")
|
|
2138
|
+
def remove_font(digest: str = FastAPIPath(pattern=r"^[a-f0-9]{64}$")):
|
|
2139
|
+
return studio.remove_font(digest)
|
|
2140
|
+
|
|
2141
|
+
@app.post("/api/uploads", status_code=201)
|
|
2142
|
+
async def upload(
|
|
2143
|
+
request: Request,
|
|
2144
|
+
filename: str = Query(..., min_length=1, max_length=255),
|
|
2145
|
+
language: str = "sorani",
|
|
2146
|
+
languages: Annotated[
|
|
2147
|
+
list[Literal["ckb", "kmr", "ar", "en"]] | None, Query(min_length=1, max_length=4)
|
|
2148
|
+
] = None,
|
|
2149
|
+
):
|
|
2150
|
+
return await studio.upload(request, filename, language, languages)
|
|
2151
|
+
|
|
2152
|
+
@app.delete("/api/uploads/{identifier}")
|
|
2153
|
+
def remove_upload(identifier: str):
|
|
2154
|
+
return studio.remove_upload(identifier)
|
|
2155
|
+
|
|
2156
|
+
@app.post("/api/preview")
|
|
2157
|
+
def preview(request: PreviewRequest):
|
|
2158
|
+
return studio.preview(request)
|
|
2159
|
+
|
|
2160
|
+
@app.post("/api/capacity")
|
|
2161
|
+
def capacity(request: CapacityRequest):
|
|
2162
|
+
return studio.capacity(request)
|
|
2163
|
+
|
|
2164
|
+
@app.post("/api/jobs", status_code=202)
|
|
2165
|
+
def start(request: JobRequest):
|
|
2166
|
+
return studio.start(request)
|
|
2167
|
+
|
|
2168
|
+
@app.get("/api/jobs")
|
|
2169
|
+
def jobs():
|
|
2170
|
+
return {"jobs": studio.jobs()}
|
|
2171
|
+
|
|
2172
|
+
@app.get("/api/jobs/{identifier}")
|
|
2173
|
+
def job(identifier: str):
|
|
2174
|
+
return studio.job(identifier)
|
|
2175
|
+
|
|
2176
|
+
@app.post("/api/jobs/{identifier}/cancel")
|
|
2177
|
+
def cancel(identifier: str):
|
|
2178
|
+
return studio.cancel(identifier)
|
|
2179
|
+
|
|
2180
|
+
@app.get("/api/jobs/{identifier}/samples")
|
|
2181
|
+
def samples(identifier: str, offset: int = Query(0, ge=0), limit: int = Query(12, ge=1, le=24)):
|
|
2182
|
+
return studio.samples(studio.job(identifier), offset, limit)
|
|
2183
|
+
|
|
2184
|
+
@app.get("/api/jobs/{identifier}/images/{filename}")
|
|
2185
|
+
def image(identifier: str, filename: str):
|
|
2186
|
+
path = studio.image_path(studio.job(identifier), filename)
|
|
2187
|
+
if path.suffix == ".tif":
|
|
2188
|
+
# Browsers cannot display TIFF consistently; keep the original file in downloads.
|
|
2189
|
+
with Image.open(path) as source:
|
|
2190
|
+
if source.width * source.height > 16_777_216:
|
|
2191
|
+
raise HTTPException(400, "Image exceeds the supported preview size")
|
|
2192
|
+
with io.BytesIO() as buffer:
|
|
2193
|
+
source.save(buffer, format="PNG")
|
|
2194
|
+
return Response(buffer.getvalue(), media_type="image/png")
|
|
2195
|
+
return FileResponse(path)
|
|
2196
|
+
|
|
2197
|
+
@app.get("/api/jobs/{identifier}/thumbnails/{filename}")
|
|
2198
|
+
def thumbnail(identifier: str, filename: str):
|
|
2199
|
+
return studio.thumbnail(studio.job(identifier), filename)
|
|
2200
|
+
|
|
2201
|
+
@app.post("/api/jobs/{identifier}/download")
|
|
2202
|
+
def prepare_download(
|
|
2203
|
+
identifier: str,
|
|
2204
|
+
format: DOWNLOAD_FORMAT = "dataset",
|
|
2205
|
+
):
|
|
2206
|
+
# Check and prepare before triggering the native browser download. Errors
|
|
2207
|
+
# remain in the form instead of replacing it with a JSON error page.
|
|
2208
|
+
try:
|
|
2209
|
+
return studio.download(studio.job(identifier), format, prepare_only=True)
|
|
2210
|
+
except HTTPException as error:
|
|
2211
|
+
if error.status_code == 413:
|
|
2212
|
+
raise HTTPException(
|
|
2213
|
+
413,
|
|
2214
|
+
f"This ZIP exceeds the {studio.cache_limit // 1048576} MiB download limit. "
|
|
2215
|
+
"Try fewer images or JPEG format in a new run. "
|
|
2216
|
+
"You can still download the JSONL labels.",
|
|
2217
|
+
) from error
|
|
2218
|
+
if error.status_code == 503:
|
|
2219
|
+
raise HTTPException(
|
|
2220
|
+
503, "Other downloads are in progress. Try again after they finish."
|
|
2221
|
+
) from error
|
|
2222
|
+
raise
|
|
2223
|
+
|
|
2224
|
+
@app.get("/api/jobs/{identifier}/download")
|
|
2225
|
+
def download(
|
|
2226
|
+
identifier: str,
|
|
2227
|
+
format: DOWNLOAD_FORMAT = "dataset",
|
|
2228
|
+
):
|
|
2229
|
+
return studio.download(studio.job(identifier), format)
|
|
2230
|
+
|
|
2231
|
+
def ui_font(name: str):
|
|
2232
|
+
fonts = {
|
|
2233
|
+
"ui-latin.ttf": "fonts/open/notosans/NotoSans[wdth,wght].ttf",
|
|
2234
|
+
"ui-kurdish.ttf": "fonts/open/notonaskharabic/NotoNaskhArabic[wght].ttf",
|
|
2235
|
+
}
|
|
2236
|
+
if name not in fonts:
|
|
2237
|
+
raise HTTPException(404, "Font not found")
|
|
2238
|
+
path = config.path(fonts[name])
|
|
2239
|
+
if not path.is_file():
|
|
2240
|
+
raise HTTPException(404, "Font not installed")
|
|
2241
|
+
return FileResponse(path, media_type="font/ttf")
|
|
2242
|
+
|
|
2243
|
+
@app.get("/assets/ui-latin.ttf")
|
|
2244
|
+
def latin_font():
|
|
2245
|
+
return ui_font("ui-latin.ttf")
|
|
2246
|
+
|
|
2247
|
+
@app.get("/assets/ui-kurdish.ttf")
|
|
2248
|
+
def kurdish_font():
|
|
2249
|
+
return ui_font("ui-kurdish.ttf")
|
|
2250
|
+
|
|
2251
|
+
static = Path(str(files("synthocr").joinpath("static")))
|
|
2252
|
+
if static.is_dir():
|
|
2253
|
+
app.mount("/assets", StaticFiles(directory=static), name="assets")
|
|
2254
|
+
|
|
2255
|
+
@app.get("/")
|
|
2256
|
+
def index():
|
|
2257
|
+
return FileResponse(static / "index.html")
|
|
2258
|
+
|
|
2259
|
+
return app
|