veriscript 1.5.0__py3-none-any.whl

deva_crnn/train.py

@@ -0,0 +1,254 @@
+"""
+train.py — CRNN+CTC trainer for Devanagari lines.
+
+Runs locally (CPU, tiny data) and on Vertex AI (T4, prebuilt PyTorch
+container). Data is an npz produced by `deva_crnn.data.export_npz` or by
+`scripts/export_training_data.py`.
+
+Usage:
+    python -m deva_crnn.train --data data.npz --out out/deva_crnn \
+        --epochs 30 --batch 64 --lr 3e-4
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import tempfile
+import time
+from typing import Dict, List, Optional, Tuple
+
+import numpy as np
+import torch
+import torch.nn as nn
+from torch.utils.data import DataLoader, Dataset
+
+from .charset import build_charset, decode, encode
+from .data import IN_H, IN_W, load_npz
+from .model import CRNN
+
+
+def resolve_data_path(path: str) -> str:
+    """`pkg://data/x.npz` resolves inside the installed package (Vertex jobs
+    ship the npz inside the wheel; local runs use plain paths)."""
+    if path.startswith("pkg://"):
+        import importlib.resources as ir
+        return str(ir.files("deva_crnn").joinpath(path[len("pkg://"):]))
+    return path
+
+
+class LineDataset(Dataset):
+    def __init__(self, images: np.ndarray, texts: List[str], charset: List[str]):
+        self.images = images
+        self.texts = texts
+        self.charset = charset
+
+    def __len__(self) -> int:
+        return len(self.texts)
+
+    def __getitem__(self, i: int):
+        img = torch.from_numpy(self.images[i].astype(np.float32) / 255.0)
+        img = (img - 0.5) / 0.5
+        target = torch.tensor(encode(self.texts[i], self.charset),
+                              dtype=torch.long)
+        return img.unsqueeze(0), target, len(target)
+
+
+def _collate(batch):
+    imgs = torch.stack([b[0] for b in batch])
+    targets = torch.cat([b[1] for b in batch])
+    lens = torch.tensor([b[2] for b in batch], dtype=torch.long)
+    return imgs, targets, lens
+
+
+def evaluate(model: CRNN, loader: DataLoader, charset: List[str],
+             device: torch.device, max_batches: int = 0) -> Dict:
+    model.eval()
+    exact = total = 0
+    with torch.no_grad():
+        for i, (imgs, _targets, _lens) in enumerate(loader):
+            if max_batches and i >= max_batches:
+                break
+            logits = model(imgs.to(device))          # T, B, C
+            preds = logits.argmax(-1).permute(1, 0)  # B, T
+            for b, p in enumerate(preds):
+                hyp = decode(p.tolist(), charset)
+                gt = loader.dataset.texts[total]
+                exact += int(hyp == gt)
+                total += 1
+    return {"lines": total, "exact_match": (exact / total) if total else 0.0}
+
+
+def _upload_if_gcs(local_path: str, out_dir: str) -> None:
+    """Checkpoint to GCS from a Vertex job (best-effort, two backends)."""
+    if not out_dir.startswith("gs://"):
+        return
+    bucket, _, prefix = out_dir[len("gs://"):].partition("/")
+    blob = f"{prefix}/{os.path.basename(local_path)}".lstrip("/")
+    try:
+        from google.cloud import storage
+        storage.Client().bucket(bucket).blob(blob).upload_from_filename(local_path)
+        return
+    except Exception:  # noqa: BLE001
+        pass
+    import subprocess
+    try:
+        subprocess.run(["gcloud", "storage", "cp", local_path,
+                        f"gs://{bucket}/{blob}"], check=True,
+                       capture_output=True, timeout=300)
+    except Exception as e:  # noqa: BLE001
+        print(f"WARN: could not upload {local_path} to GCS: {e}", flush=True)
+
+
+def resolve_out_dirs(out_dir: str) -> Tuple[str, Optional[str]]:
+    """(local_dir, remote_prefix) for a trainer output path.
+
+    `gs://bucket/prefix` keeps the per-epoch files in a local temp dir and
+    uploads each one via `_upload_if_gcs` (the Vertex/GCE pattern); a plain
+    path is used as-is with no upload. Before this, `gs://` outputs were
+    written to a literal `gs:/...` directory and the upload fallback copied a
+    non-existent GCS object to itself - checkpoints never left the VM.
+    """
+    if out_dir.startswith("gs://"):
+        return os.path.join(tempfile.gettempdir(), "deva_crnn_out"), out_dir
+    return out_dir, None
+
+
+def train(data_path: str, out_dir: str, epochs: int = 30, batch: int = 64,
+          lr: float = 1e-3, val_split: float = 0.05, seed: int = 1,
+          max_hours: float = 3.0, init: Optional[str] = None,
+          workers: int = -1, in_h: int = IN_H, in_w: int = IN_W,
+          hidden: int = 256, lr_schedule: str = "none",
+          save_best: bool = False) -> Dict:
+    images, texts = load_npz(resolve_data_path(data_path))
+    if images.shape[1] != in_h or images.shape[2] != in_w:
+        raise SystemExit(
+            f"data shape {images.shape[1]}x{images.shape[2]} != --in-h "
+            f"{in_h} x --in-w {in_w}; re-export the npz with --height "
+            f"{in_h} --width {in_w}")
+    charset = build_charset(texts)
+    rng = np.random.default_rng(seed)
+    order = rng.permutation(len(texts))
+    n_val = max(1, int(len(texts) * val_split))
+    val_idx, train_idx = order[:n_val], order[n_val:]
+    ds_tr = LineDataset(images[train_idx], [texts[i] for i in train_idx], charset)
+    ds_va = LineDataset(images[val_idx], [texts[i] for i in val_idx], charset)
+    # Worker processes cost ~3 s per epoch on Windows regardless of dataset
+    # size; only worth it for real datasets. `workers=0` avoids torch's shared
+    # file mapping, which fails on memory-constrained Windows boxes.
+    if workers < 0:
+        workers = 0 if len(ds_tr) < 512 else 2
+    dl_tr = DataLoader(ds_tr, batch_size=batch, shuffle=True, collate_fn=_collate,
+                       num_workers=workers,
+                       drop_last=len(ds_tr) >= batch)
+    dl_va = DataLoader(ds_va, batch_size=batch, shuffle=False,
+                       collate_fn=_collate)
+
+    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+    model = CRNN(n_classes=len(charset), hidden=hidden,
+                 in_h=in_h).to(device)
+    if init:
+        ck = torch.load(init, map_location=device, weights_only=False)
+        if list(ck["charset"]) != list(charset):
+            raise SystemExit(
+                f"--init charset mismatch ({len(ck['charset'])} vs "
+                f"{len(charset)}): rebuild the data with --match-charset")
+        model.load_state_dict(ck["model"])
+        print(f"initialized from {init} (epoch {ck.get('epoch')})", flush=True)
+    opt = torch.optim.AdamW(model.parameters(), lr=lr)
+    if lr_schedule == "cosine":
+        sched = torch.optim.lr_scheduler.CosineAnnealingLR(opt, T_max=max(1, epochs))
+    else:
+        sched = None
+    ctc = nn.CTCLoss(blank=0, zero_infinity=True)
+
+    local_dir, remote = resolve_out_dirs(out_dir)
+    os.makedirs(local_dir, exist_ok=True)
+    history: List[Dict] = []
+    best: Optional[float] = None
+    t0 = time.time()
+    step = 0
+    for epoch in range(epochs):
+        model.train()
+        losses = []
+        for imgs, targets, lens in dl_tr:
+            logits = model(imgs.to(device))          # T, B, C
+            T = logits.shape[0]
+            input_lens = torch.full((imgs.shape[0],), T, dtype=torch.long)
+            loss = ctc(logits, targets, input_lens, lens)
+            if not torch.isfinite(loss):
+                opt.zero_grad(set_to_none=True)
+                continue  # never let one bad batch poison the weights
+            opt.zero_grad()
+            loss.backward()
+            torch.nn.utils.clip_grad_norm_(model.parameters(), 5.0)
+            opt.step()
+            losses.append(float(loss.item()))
+            step += 1
+        val = evaluate(model, dl_va, charset, device)
+        rec = {"epoch": epoch + 1,
+               "loss": float(np.mean(losses)) if losses else None, **val,
+               "seconds": round(time.time() - t0, 1)}
+        history.append(rec)
+        print(f"epoch {epoch + 1}/{epochs} loss {rec['loss']:.3f} "
+              f"val_exact {val['exact_match']:.3f} ({rec['seconds']}s)",
+              flush=True)
+        ckpt_path = os.path.join(local_dir, "ckpt.pt")
+        payload = {"model": model.state_dict(), "charset": charset,
+                   "in_h": in_h, "in_w": in_w, "epoch": epoch + 1}
+        torch.save(payload, ckpt_path)
+        best_path = os.path.join(local_dir, "ckpt_best.pt")
+        if save_best and (best is None or val["exact_match"] > best):
+            best = val["exact_match"]
+            torch.save(payload, best_path)
+            print(f"  new best val_exact {best:.3f} -> ckpt_best.pt", flush=True)
+        if sched is not None:
+            sched.step()
+        metrics_path = os.path.join(local_dir, "metrics.json")
+        with open(metrics_path, "w") as f:
+            json.dump({"history": history, "charset_size": len(charset),
+                       "lines": len(texts), "device": str(device)}, f, indent=2)
+        if remote:
+            _upload_if_gcs(ckpt_path, remote)
+            if save_best:
+                _upload_if_gcs(best_path, remote)
+            _upload_if_gcs(metrics_path, remote)
+        if (time.time() - t0) / 3600.0 > max_hours:
+            print("time budget reached, stopping", flush=True)
+            break
+    return {"history": history, "out_dir": out_dir}
+
+
+def main():
+    ap = argparse.ArgumentParser(description="CRNN+CTC Devanagari line trainer")
+    ap.add_argument("--data", required=True)
+    ap.add_argument("--out", default="out/deva_crnn")
+    ap.add_argument("--epochs", type=int, default=30)
+    ap.add_argument("--batch", type=int, default=64)
+    ap.add_argument("--lr", type=float, default=1e-3)
+    ap.add_argument("--seed", type=int, default=1)
+    ap.add_argument("--max-hours", type=float, default=3.0)
+    ap.add_argument("--init", default=None,
+                    help="checkpoint to warm-start from (charset must match)")
+    ap.add_argument("--workers", type=int, default=-1,
+                    help="DataLoader workers (-1 auto; 0 avoids shared memory)")
+    ap.add_argument("--in-h", type=int, default=IN_H,
+                    help="input height the npz was exported at (32 or 48)")
+    ap.add_argument("--in-w", type=int, default=IN_W,
+                    help="input width the npz was exported at (256 or 512)")
+    ap.add_argument("--hidden", type=int, default=256,
+                    help="BiLSTM hidden size (256 or 384)")
+    ap.add_argument("--lr-schedule", choices=("none", "cosine"), default="none",
+                    help="cosine decays lr to ~0 over the run (fine-tunes)")
+    ap.add_argument("--save-best", action="store_true",
+                    help="also write ckpt_best.pt on val_exact improvements")
+    args = ap.parse_args()
+    train(args.data, args.out, epochs=args.epochs, batch=args.batch,
+          lr=args.lr, seed=args.seed, max_hours=args.max_hours,
+          init=args.init, workers=args.workers, in_h=args.in_h,
+          in_w=args.in_w, hidden=args.hidden,
+          lr_schedule=args.lr_schedule, save_best=args.save_best)
+
+
+if __name__ == "__main__":
+    main()

veriscript/__init__.py

@@ -0,0 +1,9 @@
+"""VeriScript — offline document restoration (Devanagari first)."""
+from importlib.metadata import PackageNotFoundError, version as _package_version
+
+try:
+    __version__ = _package_version("veriscript")
+except PackageNotFoundError:  # source checkout without an installed distribution
+    __version__ = "0+unknown"
+
+__all__ = ["__version__"]

veriscript/__main__.py

@@ -0,0 +1,4 @@
+from veriscript.cli import main
+
+if __name__ == "__main__":
+    main()

veriscript/branding.py

@@ -0,0 +1,26 @@
+"""
+branding.py — product name and environment-variable compatibility.
+
+The product is **VeriScript**. Current deployments read `VERISCRIPT_*`
+environment variables; the former name's `VECTOVECTO_*` variables keep
+working as a fallback, so existing shells, Docker commands and hosted
+deployments do not need to change.
+
+No heavy imports; safe for the shipped path.
+"""
+from __future__ import annotations
+
+import os
+
+NAME = "VeriScript"
+ENV_PREFIX = "VERISCRIPT_"
+LEGACY_NAME = "VectoVecto"
+LEGACY_ENV_PREFIX = "VECTOVECTO_"
+
+
+def env(name: str, default=None):
+    """`VERISCRIPT_<name>` when set, else `VECTOVECTO_<name>`, else default."""
+    value = os.environ.get(ENV_PREFIX + name)
+    if value is None:
+        value = os.environ.get(LEGACY_ENV_PREFIX + name)
+    return default if value is None else value

veriscript/calibration.py

@@ -0,0 +1,74 @@
+"""
+calibration.py — confidence calibration maps (loaded by document_ocr).
+
+The fitted file lives in `calibration/` (source checkout) or the wheel
+data-files location `<sys.prefix>/calibration/`; it is produced by
+fit_calibration.py on a *development* set (never a frozen eval set). Only
+monotone (isotonic) maps are shipped: temperature scaling was measured to be
+structurally unsuitable for the Devanagari letterpress domain (saturated
+probabilities; BCE optimum inverted the ranking, T<0).
+
+No heavy imports; safe for the shipped path.
+"""
+from __future__ import annotations
+
+import json
+import os
+import sys
+from typing import Dict, Optional
+
+from veriscript.paths import ROOT
+
+BASE_DIR = ROOT
+CALIBRATION_NAME = "rapidocr_devanagari_v1.json"
+DEFAULT_CALIBRATION = os.path.join(BASE_DIR, "calibration", CALIBRATION_NAME)
+
+_CACHE: Dict[str, Dict] = {}
+
+
+def resolve_calibration(path: Optional[str] = None) -> Optional[str]:
+    """First existing calibration file, or None.
+
+    `path` (when given) is exact; otherwise the repo copy and the wheel
+    data-files location are probed in order.
+    """
+    if path:
+        return os.path.abspath(path) if os.path.isfile(path) else None
+    for cand in (DEFAULT_CALIBRATION,
+                 os.path.join(sys.prefix, "calibration", CALIBRATION_NAME)):
+        if os.path.isfile(cand):
+            return os.path.abspath(cand)
+    return None
+
+
+def load_calibration(path: Optional[str] = None) -> Optional[Dict]:
+    """Load and cache a calibration JSON; None if missing or malformed."""
+    resolved = resolve_calibration(path)
+    if resolved is None:
+        return None
+    if resolved in _CACHE:
+        return _CACHE[resolved]
+    try:
+        with open(resolved, encoding="utf-8") as f:
+            data = json.load(f)
+        if "isotonic" not in data:
+            return None
+    except Exception:  # noqa: BLE001
+        return None
+    _CACHE[resolved] = data
+    return data
+
+
+def apply_isotonic(conf_pct: float, mapping: Dict) -> float:
+    """Map a 0-100 confidence through the piecewise-constant isotonic table."""
+    xs, ys = mapping.get("x", []), mapping.get("y", [])
+    if not xs or not ys:
+        return conf_pct
+    x = conf_pct / 100.0
+    y = ys[0]
+    for xi, yi in zip(xs, ys):
+        if x >= xi:
+            y = yi
+        else:
+            break
+    return max(0.0, min(100.0, y * 100.0))