synthocr 2.1.0__py3-none-any.whl

synthocr/corpora.py

@@ -0,0 +1,160 @@
+"""Versioned Tatoeba imports with author attribution and reproducible source hashes."""
+
+import bz2
+import json
+import logging
+import os
+import re
+import tempfile
+from datetime import date
+from pathlib import Path
+
+from .corpus_store import LEGACY_MACROLANGUAGES, canonical_language
+from .downloads import download, sha256, within
+
+LOG = logging.getLogger(__name__)
+LICENSE = "CC-BY-2.0-FR"
+LICENSE_URL = "https://creativecommons.org/licenses/by/2.0/fr/"
+
+
+def _convert(
+    archive: Path, target: Path, language: str, limit: int, conversion_version: int = 1
+) -> dict:
+    count = unattributed = 0
+    target.parent.mkdir(parents=True, exist_ok=True)
+    temporary = None
+    try:
+        with tempfile.NamedTemporaryFile(
+            mode="w", encoding="utf-8", dir=target.parent, delete=False
+        ) as output:
+            temporary = Path(output.name)
+            with bz2.open(archive, "rt", encoding="utf-8", errors="strict") as source:
+                while line := source.readline(1_000_001):
+                    if len(line) > 1_000_000:
+                        raise ValueError("Tatoeba source line exceeds 1,000,000 characters")
+                    fields = line.rstrip("\r\n").split("\t")
+                    if len(fields) != 6 or fields[1] != language or not fields[0].isdigit():
+                        raise ValueError("Unexpected Tatoeba detailed TSV record")
+                    identifier, _, text, author, added, modified = fields
+                    if author in {"", "\\N"}:
+                        unattributed += 1
+                        continue
+                    row = {
+                        "text": text,
+                        "language": (
+                            LEGACY_MACROLANGUAGES.get(language, canonical_language(language))
+                            if conversion_version in {1, 2}
+                            else canonical_language(language)
+                        ),
+                        "genre": "dialogue",
+                        "attribution": {
+                            "source": "Tatoeba",
+                            "source_id": identifier,
+                            "url": f"https://tatoeba.org/en/sentences/show/{identifier}",
+                            "author": author,
+                            "license": LICENSE,
+                            "license_url": LICENSE_URL,
+                            "date_added": added,
+                            "date_modified": modified,
+                            "modified": False,
+                        },
+                    }
+                    if conversion_version >= 2:
+                        row["attribution"]["source_language"] = language
+                    output.write(json.dumps(row, ensure_ascii=False) + "\n")
+                    count += 1
+                    if count == limit:
+                        break
+        if count == 0:
+            raise ValueError(f"No attributed Tatoeba records found for {language}")
+        os.link(temporary, target)
+    finally:
+        if temporary is not None:
+            temporary.unlink(missing_ok=True)
+    return {"records": count, "skipped_unattributed": unattributed, "raw_sha256": sha256(target)}
+
+
+def snapshot(
+    raw: Path,
+    catalog: Path,
+    cache: Path,
+    languages: list[str],
+    snapshot_date: str,
+    limit: int = 10000,
+) -> dict:
+    date.fromisoformat(snapshot_date)
+    if not re.fullmatch(r"\d{4}-\d{2}-\d{2}", snapshot_date) or limit < 1:
+        raise ValueError("Use an ISO snapshot date and a positive record limit")
+    if not languages or any(not re.fullmatch("[a-z]{3}", lang) for lang in languages):
+        raise ValueError("Languages must use ISO 639-3 codes, for example ckb or eng")
+    document = (
+        json.loads(catalog.read_text(encoding="utf-8"))
+        if catalog.exists()
+        else {"schema_version": 1, "sources": []}
+    )
+    result = []
+    for language in dict.fromkeys(languages):
+        key = f"tatoeba-{snapshot_date}-{language}"
+        existing = next((s for s in document["sources"] if s["id"] == key), None)
+        relative = f"tatoeba/{snapshot_date}/{language}.jsonl"
+        target = within(raw, relative)
+        if existing:
+            if existing["limit"] != limit:
+                raise ValueError("Snapshot limit changed; choose a new snapshot date")
+            if target.is_file() and sha256(target) == existing["raw_sha256"]:
+                result.append(existing | {"cached": True})
+                continue
+            if target.exists():
+                raise ValueError(f"Raw snapshot has changed: {target}; it will not be overwritten")
+        elif target.exists():
+            raise ValueError(f"Unregistered raw data already exists: {target}")
+        url = (
+            f"https://downloads.tatoeba.org/exports/per_language/{language}/"
+            f"{language}_sentences_detailed.tsv.bz2"
+        )
+        archive = cache / snapshot_date / f"{language}.tsv.bz2"
+        digest = download(
+            url,
+            existing["archive_sha256"] if existing else None,
+            archive,
+            128 * 1024 * 1024,
+            {"downloads.tatoeba.org"},
+        )
+        conversion_version = existing.get("conversion_version", 1) if existing else 3
+        converted = _convert(archive, target, language, limit, conversion_version)
+        if existing and converted["raw_sha256"] != existing["raw_sha256"]:
+            # Only remove this failed reproduction, never a preexisting raw snapshot.
+            target.unlink()
+            raise ValueError("Converted snapshot differs from its catalog checksum")
+        entry = {
+            "id": key,
+            "provider": "Tatoeba",
+            "language": language,
+            "language_tag": canonical_language(language),
+            "conversion_version": conversion_version,
+            "snapshot": snapshot_date,
+            "url": url,
+            "archive_sha256": digest,
+            "output": relative,
+            "limit": limit,
+            "license": LICENSE,
+            "license_url": LICENSE_URL,
+            **converted,
+        }
+        if not existing:
+            document["sources"].append(entry)
+        catalog.parent.mkdir(parents=True, exist_ok=True)
+        with tempfile.NamedTemporaryFile(
+            mode="w", encoding="utf-8", dir=catalog.parent, delete=False
+        ) as stream:
+            json.dump(document, stream, ensure_ascii=False, indent=2)
+            stream.write("\n")
+            temp_catalog = Path(stream.name)
+        os.replace(temp_catalog, catalog)
+        result.append(entry)
+        LOG.info("Imported %s: %s attributed sentences", language, entry["records"])
+    return {
+        "sources": len(result),
+        "records": sum(s["records"] for s in result),
+        "catalog": str(catalog),
+    }

synthocr/corpus_store.py

@@ -0,0 +1,222 @@
+"""Explicit source-language identities for a single corpus per language tag.
+
+Only known code aliases are folded. Script, region and dialect subtags remain
+distinct; this module never infers a language from a filename or its text.
+"""
+
+import hashlib
+import json
+import re
+import sqlite3
+from pathlib import Path
+
+LANGUAGE = re.compile(r"[A-Za-z]{2,8}(?:-[A-Za-z0-9]{1,8})*\Z")
+# These are the exact ISO 639-3 source codes used by our imported collections.
+# Keep this mapping explicit: generic Kurdish is not Sorani or Kurmanji, and
+# an unrecognised language is not guessed from a neighbouring code or script.
+ALIASES = {
+    "amh": "am",
+    "ara": "ar",
+    "aze": "az",
+    "ben": "bn",
+    "deu": "de",
+    "ell": "el",
+    "eng": "en",
+    "fra": "fr",
+    "heb": "he",
+    "hin": "hi",
+    "hye": "hy",
+    "ind": "id",
+    "ita": "it",
+    "jpn": "ja",
+    "kat": "ka",
+    "kor": "ko",
+    "nld": "nl",
+    "pol": "pl",
+    "por": "pt",
+    "rus": "ru",
+    "spa": "es",
+    "tam": "ta",
+    "tha": "th",
+    "tur": "tr",
+    "uig": "ug",
+    "ukr": "uk",
+    "urd": "ur",
+    "vie": "vi",
+}
+# These are specific languages within macrolanguages, not interchangeable
+# aliases. Historical imports used the broader tag; only explicit provenance
+# may recover the precise source identity without rewriting those raw files.
+LEGACY_MACROLANGUAGES = {"cmn": "zh", "pes": "fa", "swh": "sw"}
+
+
+def canonical_language(tag: str) -> str:
+    """Return a stable, case-insensitive identity without collapsing dialects."""
+    if not isinstance(tag, str) or (tag and not LANGUAGE.fullmatch(tag)):
+        raise ValueError("Source language must be empty or a valid language tag")
+    value = tag.casefold()
+    base, separator, subtags = value.partition("-")
+    return ALIASES.get(base, base) + separator + subtags
+
+
+def resolve_source_language(declared: str, attribution: dict, registered: str = "") -> str:
+    """Recover only reviewed historical macro/specific pairs from metadata.
+
+    Arbitrary conflicting language fields cannot override a record's declared
+    language. A generic user language remains generic without explicit source
+    metadata identifying its specific variety.
+    """
+    canonical = canonical_language(declared)
+    for candidate in (attribution.get("source_language"), registered):
+        if not isinstance(candidate, str) or not LANGUAGE.fullmatch(candidate):
+            continue
+        precise = canonical_language(candidate)
+        if LEGACY_MACROLANGUAGES.get(precise) == canonical:
+            return candidate
+    return declared
+
+
+def registered_source_languages(raw: Path, sources: list[dict]) -> dict[str, str]:
+    """Read explicit source identities only from matching pinned catalog rows.
+
+    The companion catalog is optional for mounted or custom raw collections.
+    Paths identify registered source files; they never imply a language.
+    """
+    catalog = raw.parent / "corpora.json"
+    if not catalog.is_file():
+        return {}
+    document = json.loads(catalog.read_text(encoding="utf-8"))
+    if not isinstance(document, dict) or not isinstance(document.get("sources"), list):
+        raise ValueError("Corpus source catalog must contain a sources list")
+    checksums = {source["path"]: source["sha256"] for source in sources}
+    result = {}
+    for entry in document["sources"]:
+        if not isinstance(entry, dict):
+            raise ValueError("Corpus source catalog entries must be objects")
+        path, language = entry.get("output"), entry.get("language")
+        if (
+            isinstance(path, str)
+            and isinstance(language, str)
+            and language in LEGACY_MACROLANGUAGES
+            and checksums.get(path) == entry.get("raw_sha256")
+            and path in checksums
+        ):
+            if path in result and result[path] != language:
+                raise ValueError(f"Conflicting source languages in corpus catalog: {path}")
+            result[path] = language
+    return result
+
+
+def create_origins(db: sqlite3.Connection) -> None:
+    """Store provenance separately so repeated labels never build huge arrays."""
+    db.execute(
+        "CREATE TABLE origins (id INTEGER PRIMARY KEY, text_id INTEGER NOT NULL, "
+        "source TEXT NOT NULL, line INTEGER NOT NULL, genre TEXT NOT NULL, "
+        "source_language TEXT NOT NULL, attribution TEXT, origin_hash BLOB NOT NULL, "
+        "UNIQUE(text_id,origin_hash))"
+    )
+
+
+def add_origin(
+    db: sqlite3.Connection,
+    text_id: int,
+    source: str,
+    line: int,
+    genre: str,
+    source_language: str,
+    attribution: dict,
+) -> None:
+    encoded = json.dumps(attribution, ensure_ascii=False, sort_keys=True)
+    first = db.execute(
+        "SELECT source,line,genre,attribution FROM texts WHERE id=?", (text_id,)
+    ).fetchone()
+    if first is None:
+        raise ValueError("A corpus origin must reference an existing text")
+    # The first origin reuses the attribution already stored on its text row.
+    # Repeating that JSON in every origin would almost double the database.
+    payload = (
+        None
+        if first[:3] == (source, line, genre) and json.loads(first[3]) == attribution
+        else encoded
+    )
+    digest = hashlib.sha256(
+        json.dumps(
+            [source, line, genre, source_language, attribution],
+            ensure_ascii=False,
+            sort_keys=True,
+        ).encode()
+    ).digest()
+    db.execute(
+        "INSERT OR IGNORE INTO origins"
+        "(text_id,source,line,genre,source_language,attribution,origin_hash) VALUES(?,?,?,?,?,?,?)",
+        (
+            text_id,
+            source,
+            line,
+            genre,
+            source_language,
+            payload,
+            digest,
+        ),
+    )
+
+
+def iter_origins(db: sqlite3.Connection, text_id: int | None = None):
+    """Yield all origins, or those of one label, in deterministic insertion order."""
+    where, parameters = (
+        (" WHERE origins.text_id=?", (text_id,)) if text_id is not None else ("", ())
+    )
+    cursor = db.execute(
+        "SELECT origins.text_id,origins.source,origins.line,origins.genre,source_language,"
+        "COALESCE(origins.attribution,texts.attribution) FROM origins "
+        "JOIN texts ON texts.id=origins.text_id" + where + " ORDER BY origins.text_id,origins.id",
+        parameters,
+    )
+    for identifier, source, line, genre, language, attribution in cursor:
+        yield {
+            "text_id": identifier,
+            "source": source,
+            "line": line,
+            "genre": genre,
+            "source_language": language,
+            "attribution": json.loads(attribution),
+        }
+
+
+def materialize_primary_origin(db: sqlite3.Connection, text_id: int) -> None:
+    """Preserve the old attribution before choosing a better primary origin."""
+    db.execute(
+        "UPDATE origins SET attribution=(SELECT attribution FROM texts WHERE id=?) "
+        "WHERE text_id=? AND attribution IS NULL",
+        (text_id, text_id),
+    )
+
+
+def corpus_registry(db: sqlite3.Connection, fingerprint: str, sources: list[dict]) -> dict:
+    """Describe one logical corpus per language without copying any corpus text."""
+    source_index = {source["path"]: source for source in sources}
+    corpora = []
+    for language, count, diagnostics in db.execute(
+        "SELECT language,COUNT(*),SUM(genre='spelling') FROM texts GROUP BY language "
+        "ORDER BY language"
+    ):
+        paths = (
+            row[0]
+            for row in db.execute(
+                "SELECT DISTINCT origins.source FROM origins "
+                "JOIN texts ON texts.id=origins.text_id WHERE texts.language=? "
+                "ORDER BY origins.source",
+                (language,),
+            )
+        )
+        entries = [source_index[path] for path in paths]
+        corpora.append(
+            {
+                "language": language,
+                "records": count,
+                "diagnostic_records": diagnostics,
+                "source_count": len(entries),
+                "sources": entries,
+            }
+        )
+    return {"schema_version": 1, "fingerprint": fingerprint, "corpora": corpora}

synthocr/downloads.py

@@ -0,0 +1,77 @@
+"""Bounded, checksummed downloads shared by fonts and corpus snapshots."""
+
+import hashlib
+import os
+import tempfile
+from pathlib import Path
+from urllib.parse import urlparse
+from urllib.request import urlopen
+
+
+def sha256(path: Path) -> str:
+    with path.open("rb") as stream:
+        return hashlib.file_digest(stream, "sha256").hexdigest()
+
+
+def download(
+    url: str, digest: str | None, target: Path, max_bytes: int, hosts: set[str], opener=None
+) -> str:
+    parsed = urlparse(url)
+    if (
+        parsed.scheme != "https"
+        or parsed.hostname not in hosts
+        or parsed.username
+        or parsed.password
+    ):
+        raise ValueError(f"Downloads require HTTPS on {', '.join(sorted(hosts))}")
+    if target.is_file():
+        if target.stat().st_size > max_bytes:
+            raise ValueError(f"Existing download exceeds {max_bytes} bytes: {target}")
+        actual = sha256(target)
+        if digest is None or actual == digest:
+            return actual
+        raise ValueError(f"Existing file has a different checksum: {target}")
+    target.parent.mkdir(parents=True, exist_ok=True)
+    temporary = None
+    try:
+        with (opener or urlopen)(url, timeout=60) as response:
+            final = urlparse(response.geturl()) if hasattr(response, "geturl") else parsed
+            if final.scheme != "https" or final.hostname not in hosts:
+                raise ValueError("Download redirected outside the allowed HTTPS hosts")
+            size = response.headers.get("Content-Length") if hasattr(response, "headers") else None
+            if size and int(size) > max_bytes:
+                raise ValueError(f"Download exceeds {max_bytes} bytes: {url}")
+            with tempfile.NamedTemporaryFile(dir=target.parent, delete=False) as stream:
+                temporary = Path(stream.name)
+                hasher = hashlib.sha256()
+                total = 0
+                while chunk := response.read(65536):
+                    total += len(chunk)
+                    if total > max_bytes:
+                        raise ValueError(f"Download exceeds {max_bytes} bytes: {url}")
+                    hasher.update(chunk)
+                    stream.write(chunk)
+        actual = hasher.hexdigest()
+        if digest is not None and actual != digest:
+            raise ValueError(f"SHA-256 verification failed: {url}")
+        # Temporary files are owner-only; give the asset ordinary read permissions
+        # so non-root users (such as the container's) can read it.
+        temporary.chmod(0o644)
+        # Link is atomic and refuses an existing destination, even after a race.
+        os.link(temporary, target)
+        return actual
+    finally:
+        if temporary is not None:
+            temporary.unlink(missing_ok=True)
+
+
+def within(root: Path, name: str) -> Path:
+    relative = Path(name)
+    target = (root / relative).resolve()
+    if (
+        relative.is_absolute()
+        or ".." in relative.parts
+        or not target.is_relative_to(root.resolve())
+    ):
+        raise ValueError("Catalog paths must stay inside the destination")
+    return target