asterocr 1.0.0__py3-none-any.whl

asterocr/synthesizer/mega_generator.py

@@ -0,0 +1,527 @@
+"""
+AsterOCR Extreme Synthetic Data Generator (Mega Generator)
+Generates millions of heavily degraded, real-world dirty/aged document images:
+- Stains: Coffee stains, tea rings, mold spots, deep yellow aging
+- Damage: Paper creases/folds cutting through text, torn borders, staple holes
+- Scanner flaws: Dirty roller vertical streaks, black border shadows, contrast clipping
+- Text distortions: Broken ink lines, ink bleed/blobs, motion blur, irregular handwriting
+- Multi-scale: Tiny to massive fonts, wide/narrow aspect ratio compression
+- Output: Standard images + comprehensive metadata JSON
+- Multi-processing: Utilizes all CPU cores for maximum throughput
+"""
+
+import os
+import io
+import sys
+import json
+import time
+import random
+import math
+import argparse
+from multiprocessing import Pool, cpu_count
+import numpy as np
+from PIL import Image, ImageDraw, ImageFont, ImageFilter, ImageEnhance
+import cv2
+
+try:
+    import lmdb
+except ImportError:
+    lmdb = None
+
+if sys.platform == "win32":
+    try:
+        sys.stdout.reconfigure(encoding='utf-8')
+        sys.stderr.reconfigure(encoding='utf-8')
+    except Exception:
+        pass
+
+# Ensure root in sys.path
+sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "../..")))
+from asterocr.synthesizer.corpus import generate_random_corpus_text
+from asterocr.synthesizer.generator import default_generator, SYSTEM_FONT_DIR
+
+class MegaDegradationEngine:
+    """Extreme real-world degradation effects engine."""
+    
+    @staticmethod
+    def add_paper_aging_and_stains(img):
+        w, h = img.size
+        arr = np.array(img, dtype=np.float32)
+        
+        # 1. Base yellow/brown aged tint
+        r_scale = random.uniform(0.92, 1.0)
+        g_scale = random.uniform(0.80, 0.92)
+        b_scale = random.uniform(0.60, 0.78)
+        arr[:, :, 0] *= r_scale
+        arr[:, :, 1] *= g_scale
+        arr[:, :, 2] *= b_scale
+        
+        # 2. Random coffee / tea stains or moisture rings
+        if random.random() < 0.6:
+            num_stains = random.randint(1, 3)
+            for _ in range(num_stains):
+                cx, cy = random.randint(0, w), random.randint(0, h)
+                rx, ry = random.randint(15, 60), random.randint(10, 30)
+                y_idx, x_idx = np.ogrid[:h, :w]
+                dist = ((x_idx - cx) / rx) ** 2 + ((y_idx - cy) / ry) ** 2
+                mask = np.exp(-dist * 1.5)
+                # Stain darkening factor
+                stain_color = np.array([random.uniform(0.7, 0.85), random.uniform(0.6, 0.75), random.uniform(0.4, 0.6)])
+                for c in range(3):
+                    arr[:, :, c] = arr[:, :, c] * (1 - mask * (1 - stain_color[c]))
+
+        # 3. Paper mold / mildew specks
+        if random.random() < 0.5:
+            num_specks = random.randint(10, 50)
+            for _ in range(num_specks):
+                sx, sy = random.randint(0, w - 1), random.randint(0, h - 1)
+                rad = random.randint(1, 3)
+                arr[max(0, sy-rad):min(h, sy+rad), max(0, sx-rad):min(w, sx+rad)] *= random.uniform(0.3, 0.6)
+
+        return Image.fromarray(np.clip(arr, 0, 255).astype(np.uint8))
+
+    @staticmethod
+    def add_creases_and_folds(img):
+        """Adds horizontal, diagonal or vertical paper fold creases cutting through text."""
+        if random.random() > 0.6:
+            return img
+            
+        w, h = img.size
+        arr = np.array(img, dtype=np.float32)
+        
+        # Line from one edge to another
+        x1, y1 = random.randint(0, w // 4), random.randint(0, h)
+        x2, y2 = random.randint(w * 3 // 4, w), random.randint(0, h)
+        
+        cv_img = arr.copy()
+        # Dark crease shadow
+        cv2.line(cv_img, (x1, y1), (x2, y2), (random.randint(50, 90), random.randint(45, 80), random.randint(35, 70)), 1)
+        # Highlighted fold edge
+        cv2.line(cv_img, (x1, max(0, y1 - 1)), (x2, max(0, y2 - 1)), (255, 255, 240), 1)
+        
+        # Blurring slightly into paper
+        cv_img = cv2.GaussianBlur(cv_img, (3, 3), 0.5)
+        # Blend back
+        alpha = random.uniform(0.4, 0.8)
+        arr = arr * (1 - alpha) + cv_img * alpha
+        return Image.fromarray(np.clip(arr, 0, 255).astype(np.uint8))
+
+    @staticmethod
+    def add_scanner_streaks_and_borders(img):
+        """Adds vertical roller streaks and dark photocopy borders."""
+        w, h = img.size
+        arr = np.array(img, dtype=np.float32)
+        
+        # Vertical streaks from defective scan roller
+        if random.random() < 0.45:
+            num_streaks = random.randint(1, 3)
+            for _ in range(num_streaks):
+                sx = random.randint(5, w - 5)
+                streak_w = random.randint(1, 3)
+                dark_val = random.uniform(0.4, 0.75)
+                arr[:, max(0, sx - streak_w):min(w, sx + streak_w)] *= dark_val
+
+        # Black or dark shadow border from photocopier lid
+        if random.random() < 0.4:
+            border_side = random.choice(["top", "bottom", "left", "right"])
+            depth = random.randint(3, 8)
+            if border_side == "top":
+                arr[:depth, :] *= np.linspace(0.1, 0.9, depth)[:, None, None]
+            elif border_side == "bottom":
+                arr[-depth:, :] *= np.linspace(0.9, 0.1, depth)[:, None, None]
+            elif border_side == "left":
+                arr[:, :depth] *= np.linspace(0.1, 0.9, depth)[None, :, None]
+            else:
+                arr[:, -depth:] *= np.linspace(0.9, 0.1, depth)[None, :, None]
+
+        return Image.fromarray(np.clip(arr, 0, 255).astype(np.uint8))
+
+    @staticmethod
+    def add_ink_degradations(img):
+        """Broken characters, ink smudge, bleed, heavy erosion / dilation."""
+        cv_img = np.array(img)
+        degrade_type = random.choice(["erode", "dilate", "salt_pepper", "blur", "clip"])
+        
+        if degrade_type == "erode":
+            # Thicker, smudged ink
+            k = np.ones((2, 2), np.uint8)
+            cv_img = cv2.erode(cv_img, k, iterations=random.randint(1, 2))
+        elif degrade_type == "dilate":
+            # Faint, broken, dried-out typewriter/pen ink
+            k = np.ones((2, 2), np.uint8)
+            cv_img = cv2.dilate(cv_img, k, iterations=1)
+        elif degrade_type == "salt_pepper":
+            # Heavy scanner noise
+            noise = np.random.choice([0, 1, 2], size=cv_img.shape[:2], p=[0.02, 0.02, 0.96])
+            cv_img[noise == 0] = [random.randint(10, 40)] * 3 # pepper
+            cv_img[noise == 1] = [random.randint(220, 255)] * 3 # salt
+        elif degrade_type == "blur":
+            # Motion or out-of-focus blur
+            ksize = random.choice([3, 5])
+            cv_img = cv2.GaussianBlur(cv_img, (ksize, ksize), random.uniform(0.5, 1.4))
+        else:
+            # Extreme contrast clipping (photocopy effect)
+            gray = cv2.cvtColor(cv_img, cv2.COLOR_RGB2GRAY)
+            _, thresh = cv2.threshold(gray, random.randint(120, 170), 255, cv2.THRESH_BINARY)
+            cv_img = cv2.cvtColor(thresh, cv2.COLOR_GRAY2RGB)
+            
+        return Image.fromarray(cv_img)
+
+    @staticmethod
+    def add_torn_edge_or_staple(img):
+        """Simulates torn paper borders or rusted staple holes."""
+        if random.random() > 0.35:
+            return img
+            
+        w, h = img.size
+        draw = ImageDraw.Draw(img)
+        
+        if random.random() < 0.5:
+            # Rusted staple hole
+            hx = random.randint(6, 16)
+            hy = random.randint(6, h - 6)
+            # Hole center (black) + rust ring (brown)
+            draw.ellipse([hx-3, hy-3, hx+3, hy+3], fill=(139, 69, 19))
+            draw.ellipse([hx-2, hy-2, hx+2, hy+2], fill=(20, 20, 20))
+        else:
+            # Jagged torn edge on left or right
+            side = random.choice(["left", "right"])
+            tear_w = random.randint(4, 10)
+            poly = []
+            for y in range(0, h, 4):
+                x = random.randint(0, tear_w) if side == "left" else random.randint(w - tear_w, w)
+                poly.append((x, y))
+            if side == "left":
+                poly = [(0, 0)] + poly + [(0, h)]
+            else:
+                poly = [(w, 0)] + poly + [(w, h)]
+            draw.polygon(poly, fill=(25, 25, 25))
+            
+        return img
+
+class AsterMegaSynthesizer:
+    def __init__(self, output_dir="dataset_mega"):
+        self.output_dir = output_dir
+        self.generator = default_generator
+        self.degrader = MegaDegradationEngine()
+
+    def generate_image_and_text(self, sample_id, text=None, mode="mixed"):
+        """Generates a single extreme-degradation OCR strip image (PIL) and its text."""
+        if not text:
+            text = generate_random_corpus_text(mode=mode)
+            
+        # 1. Variable Font Size & Aspect ratio
+        font_size = random.randint(16, 32)
+        font = self.generator.get_random_font(font_size)
+        
+        # 2. Render initial text
+        dummy = Image.new("RGB", (1, 1))
+        draw_dummy = ImageDraw.Draw(dummy)
+        bbox = draw_dummy.textbbox((0, 0), text, font=font)
+        tw = max(10, bbox[2] - bbox[0])
+        th = max(10, bbox[3] - bbox[1])
+        
+        padx = random.randint(8, 20)
+        pady = random.randint(4, 14)
+        cw, ch = tw + padx * 2, th + pady * 2
+        
+        bg_style = random.choice(["aged", "scan", "aged", "clean", "grid"])
+        canvas = self.generator.generate_background(cw, ch, style=bg_style)
+        draw = ImageDraw.Draw(canvas)
+        
+        # Realistic ink colors: faded blue, typewriter purple, charcoal, stamped red
+        ink = random.choice([
+            (random.randint(15, 40), random.randint(15, 40), random.randint(15, 40)),
+            (random.randint(10, 30), random.randint(30, 60), random.randint(100, 160)), # faded blue pen
+            (random.randint(60, 90), random.randint(20, 40), random.randint(70, 100)), # violet ink
+            (random.randint(70, 110), random.randint(70, 110), random.randint(70, 110))  # faded typewriter
+        ])
+        draw.text((padx - bbox[0], pady - bbox[1]), text, font=font, fill=ink)
+        
+        # 3. Apply cascade of extreme degradation effects
+        active_degradations = []
+        
+        # Effect A: Paper aging & stains
+        canvas = self.degrader.add_paper_aging_and_stains(canvas)
+        active_degradations.append("stains_and_aging")
+        
+        # Effect B: Creases & folds crossing text
+        if random.random() < 0.6:
+            canvas = self.degrader.add_creases_and_folds(canvas)
+            active_degradations.append("paper_creases")
+            
+        # Effect C: Scanner roller streaks & photocopy shadow borders
+        if random.random() < 0.55:
+            canvas = self.degrader.add_scanner_streaks_and_borders(canvas)
+            active_degradations.append("scanner_streaks")
+            
+        # Effect D: Broken ink, smear, salt-pepper, photocopy thresholding
+        canvas = self.degrader.add_ink_degradations(canvas)
+        active_degradations.append("ink_degradation")
+        
+        # Effect E: Torn edge / staple hole
+        if random.random() < 0.4:
+            canvas = self.degrader.add_torn_edge_or_staple(canvas)
+            active_degradations.append("torn_or_staple")
+            
+        # Effect F: Rotation / Shear (-3.5 to +3.5 deg)
+        if random.random() < 0.5:
+            angle = random.uniform(-3.5, 3.5)
+            canvas = canvas.rotate(angle, resample=Image.BILINEAR, expand=True, fillcolor=(235, 230, 220))
+            active_degradations.append("affine_rotation")
+
+        # 4. Standardize to H=32 preserving aspect ratio
+        w_orig, h_orig = canvas.size
+        new_w = max(24, int((32.0 / max(1, h_orig)) * w_orig))
+        final_img = canvas.resize((new_w, 32), Image.BILINEAR)
+        return final_img, text, active_degradations
+
+    def generate_sample_bytes(self, sample_id, text=None, mode="mixed"):
+        """Generates sample and compresses directly to JPEG bytes in RAM."""
+        final_img, text, _ = self.generate_image_and_text(sample_id, text=text, mode=mode)
+        
+        # Fast in-memory JPEG compression via OpenCV SIMD
+        img_np = np.array(final_img)
+        img_bgr = cv2.cvtColor(img_np, cv2.COLOR_RGB2BGR)
+        quality = random.randint(65, 92)
+        _, encoded = cv2.imencode('.jpg', img_bgr, [int(cv2.IMWRITE_JPEG_QUALITY), quality])
+        return sample_id, encoded.tobytes(), text
+
+    def generate_single_sample_file(self, sample_id, text=None, mode="mixed"):
+        """Generates and writes to disk with subfolder sharding to prevent NTFS slowdown."""
+        final_img, text, active_degradations = self.generate_image_and_text(sample_id, text=text, mode=mode)
+        
+        # Shard into subdirectories of 10,000 files each
+        shard_idx = sample_id // 10000
+        shard_dir = os.path.join(self.output_dir, "images", f"{shard_idx:04d}")
+        os.makedirs(shard_dir, exist_ok=True)
+        
+        filename = f"sample_{sample_id:08d}.jpg"
+        rel_path = f"images/{shard_idx:04d}/{filename}"
+        full_path = os.path.join(shard_dir, filename)
+        final_img.save(full_path, "JPEG", quality=random.randint(65, 90))
+        
+        meta = {
+            "id": f"{sample_id:08d}",
+            "image_path": rel_path,
+            "text": text,
+            "width": final_img.size[0],
+            "height": 32,
+            "degradations": active_degradations,
+            "difficulty": "extreme" if len(active_degradations) >= 4 else "hard"
+        }
+        return meta
+
+def _worker_chunk_lmdb(args):
+    """Worker task that processes a chunk of samples directly in RAM."""
+    start_id, count, mode = args
+    synth = AsterMegaSynthesizer()
+    chunk_data = []
+    for i in range(count):
+        sid = start_id + i
+        try:
+            sid, img_bytes, text = synth.generate_sample_bytes(sid, mode=mode)
+            chunk_data.append((sid, img_bytes, text))
+        except Exception:
+            pass
+    return chunk_data
+
+def _worker_chunk_files(args):
+    """Worker task that writes files to sharded subdirectories."""
+    start_id, count, out_dir, mode = args
+    synth = AsterMegaSynthesizer(output_dir=out_dir)
+    results = []
+    for i in range(count):
+        sid = start_id + i
+        try:
+            m = synth.generate_single_sample_file(sid, mode=mode)
+            results.append(m)
+        except Exception:
+            pass
+    return results
+
+def generate_mega_dataset_lmdb(total_samples=10000, output_dir="dataset_mega.lmdb", num_workers=None, mode="mixed"):
+    """
+    Ultra-high throughput LMDB dataset generator for millions of samples.
+    - Zero NTFS fragmentation / No freeze.
+    - Memory-mapped streaming write with atomic batch transactions.
+    - CPU multi-core parallel rendering.
+    """
+    if lmdb is None:
+        raise ImportError("Gói 'lmdb' chưa được cài đặt! Hãy chạy: pip install lmdb")
+        
+    os.makedirs(output_dir, exist_ok=True)
+    if num_workers is None:
+        num_workers = min(10, max(1, cpu_count() - 2))
+        
+    print("=" * 68)
+    print(f"  ASTER-OCR MILLION-SCALE LMDB SYNTHESIZER: {total_samples:,} MẪU")
+    print(f"  Chế độ nội dung : {mode.upper()} (Random 134+ Charset + Mã số + Hành chính)")
+    print(f"  Số luồng CPU    : {num_workers} workers song song")
+    print(f"  Định dạng xuất  : LMDB Chuẩn OCR (data.mdb)")
+    print(f"  Thư mục đích    : {output_dir}")
+    print("=" * 68)
+    
+    # Calculate map_size (max virtual capacity: ~35KB per sample, min 64MB)
+    map_size = max(1024 * 1024 * 64, int(total_samples * 35 * 1024))
+    
+    # Divide into task chunks
+    chunk_size = 500 if total_samples <= 50000 else 1000
+    tasks = []
+    curr_id = 1
+    while curr_id <= total_samples:
+        cnt = min(chunk_size, total_samples - curr_id + 1)
+        tasks.append((curr_id, cnt, mode))
+        curr_id += cnt
+        
+    total_chunks = len(tasks)
+    start_time = time.time()
+    samples_done = 0
+    commit_interval = 5000 # Commit to LMDB every 5000 samples
+    
+    env = lmdb.open(output_dir, map_size=map_size)
+    txn = env.begin(write=True)
+    uncommitted = 0
+    
+    try:
+        with Pool(num_workers) as pool:
+            for chunk_data in pool.imap_unordered(_worker_chunk_lmdb, tasks, chunksize=1):
+                for sid, img_bytes, text in chunk_data:
+                    image_key = f"image-{sid:09d}".encode("utf-8")
+                    label_key = f"label-{sid:09d}".encode("utf-8")
+                    
+                    txn.put(image_key, img_bytes)
+                    txn.put(label_key, text.encode("utf-8"))
+                    
+                    samples_done += 1
+                    uncommitted += 1
+                    
+                    if uncommitted >= commit_interval:
+                        txn.commit()
+                        txn = env.begin(write=True)
+                        uncommitted = 0
+                
+                # Progress display
+                elapsed = max(0.001, time.time() - start_time)
+                speed = samples_done / elapsed
+                rem = max(0, total_samples - samples_done)
+                eta_sec = rem / max(1.0, speed)
+                eta_str = time.strftime('%H:%M:%S', time.gmtime(eta_sec))
+                
+                mdb_path = os.path.join(output_dir, "data.mdb")
+                db_mb = os.path.getsize(mdb_path) / (1024 * 1024) if os.path.exists(mdb_path) else 0
+                pct = (samples_done / total_samples) * 100
+                
+                sys.stdout.write(
+                    f"\r[*] Tiến độ: {samples_done:,}/{total_samples:,} ({pct:.1f}%) | "
+                    f"Tốc độ: {speed:,.0f} mẫu/s | ETA: {eta_str} | LMDB: {db_mb:,.1f} MB  "
+                )
+                sys.stdout.flush()
+                
+        # Final commit
+        txn.put(b"num-samples", str(samples_done).encode("utf-8"))
+        txn.commit()
+    finally:
+        env.sync()
+        env.close()
+        
+    total_elapsed = time.time() - start_time
+    avg_speed = samples_done / max(0.001, total_elapsed)
+    final_size_mb = os.path.getsize(os.path.join(output_dir, "data.mdb")) / (1024 * 1024)
+    
+    print("\n" + "=" * 68)
+    print(f"[+] HOÀN THÀNH XUẤT SẮC!")
+    print(f"[+] Tổng số mẫu đã lưu vào LMDB: {samples_done:,} mẫu")
+    print(f"[+] Thời gian thực hiện        : {total_elapsed:.1f} giây (~{total_elapsed/60:.1f} phút)")
+    print(f"[+] Tốc độ trung bình          : {avg_speed:,.0f} mẫu/giây")
+    print(f"[+] Dung lượng LMDB cuối cùng  : {final_size_mb:,.2f} MB")
+    print(f"[+] Đường dẫn database         : {output_dir}")
+    print("=" * 68)
+    
+    # Save a metadata info file
+    info = {
+        "format": "lmdb",
+        "num_samples": samples_done,
+        "mode": mode,
+        "target_height": 32,
+        "size_mb": final_size_mb,
+        "created_at": time.strftime("%Y-%m-%d %H:%M:%S")
+    }
+    with open(os.path.join(output_dir, "dataset_info.json"), "w", encoding="utf-8") as f:
+        json.dump(info, f, ensure_ascii=False, indent=2)
+        
+    return output_dir
+
+def generate_mega_dataset_files(total_samples=10000, output_dir="dataset_mega", num_workers=None, mode="mixed"):
+    """Generates samples saving into sharded subdirectories."""
+    os.makedirs(output_dir, exist_ok=True)
+    if num_workers is None:
+        num_workers = min(10, max(1, cpu_count() - 2))
+        
+    print("=" * 68)
+    print(f"  ASTER-OCR SHARDED FILES SYNTHESIZER: {total_samples:,} MẪU")
+    print(f"  Số luồng CPU  : {num_workers} workers song song")
+    print(f"  Thư mục xuất  : {output_dir}")
+    print("=" * 68)
+    
+    chunk_size = 500 if total_samples <= 20000 else 1000
+    tasks = []
+    curr_id = 1
+    while curr_id <= total_samples:
+        cnt = min(chunk_size, total_samples - curr_id + 1)
+        tasks.append((curr_id, cnt, output_dir, mode))
+        curr_id += cnt
+        
+    all_metadata = []
+    start_time = time.time()
+    
+    with Pool(num_workers) as p:
+        for chunk in p.imap_unordered(_worker_chunk_files, tasks, chunksize=1):
+            all_metadata.extend(chunk)
+            samples_done = len(all_metadata)
+            elapsed = max(0.001, time.time() - start_time)
+            speed = samples_done / elapsed
+            pct = (samples_done / total_samples) * 100
+            sys.stdout.write(f"\r[*] Đã sinh: {samples_done:,}/{total_samples:,} ({pct:.1f}%) | Tốc độ: {speed:,.0f} mẫu/s  ")
+            sys.stdout.flush()
+            
+    txt_annotation_path = os.path.join(output_dir, "train_labels.txt")
+    json_annotation_path = os.path.join(output_dir, "labels.json")
+    
+    with open(json_annotation_path, "w", encoding="utf-8") as f:
+        json.dump(all_metadata, f, ensure_ascii=False, indent=2)
+        
+    with open(txt_annotation_path, "w", encoding="utf-8") as f:
+        for m in all_metadata:
+            f.write(f"{m['image_path']}\t{m['text']}\n")
+            
+    print(f"\n[+] Đã sinh {len(all_metadata):,} mẫu ảnh thành công!")
+    print(f"[+] File nhãn Text: {txt_annotation_path}")
+    return json_annotation_path
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="AsterOCR Million-Scale Synthetic Generator")
+    parser.add_argument("--num_samples", type=int, default=1000, help="Số mẫu cần sinh (Ví dụ: 100000, 1000000)")
+    parser.add_argument("--format", type=str, default="lmdb", choices=["lmdb", "files"], help="Định dạng lưu: 'lmdb' (khuyên dùng) hoặc 'files'")
+    parser.add_argument("--output_dir", type=str, default=None, help="Thư mục xuất dữ liệu (Mặc định: dataset_mega.lmdb hoặc dataset_mega)")
+    parser.add_argument("--workers", type=int, default=10, help="Số tiến trình CPU song song (Mặc định: 10 để tránh lag máy)")
+    parser.add_argument("--mode", type=str, default="mixed", choices=["mixed", "pure_random", "admin"], help="Chế độ chuỗi ký tự")
+    args = parser.parse_args()
+    
+    if args.output_dir is None:
+        args.output_dir = "dataset_mega.lmdb" if args.format == "lmdb" else "dataset_mega"
+        
+    if args.format == "lmdb":
+        generate_mega_dataset_lmdb(
+            total_samples=args.num_samples,
+            output_dir=args.output_dir,
+            num_workers=args.workers,
+            mode=args.mode
+        )
+    else:
+        generate_mega_dataset_files(
+            total_samples=args.num_samples,
+            output_dir=args.output_dir,
+            num_workers=args.workers,
+            mode=args.mode
+        )

asterocr/trainer/metrics.py

@@ -0,0 +1,53 @@
+"""
+AsterOCR Performance Evaluation Metrics
+Calculates Character Error Rate (CER) and Word Error Rate (WER) using Levenshtein distance.
+"""
+
+def levenshtein_distance(ref, hyp):
+    """Calculates edit distance between two sequences (characters or words)."""
+    m, n = len(ref), len(hyp)
+    dp = [[0] * (n + 1) for _ in range(m + 1)]
+    
+    for i in range(m + 1):
+        dp[i][0] = i
+    for j in range(n + 1):
+        dp[0][j] = j
+        
+    for i in range(1, m + 1):
+        for j in range(1, n + 1):
+            if ref[i - 1] == hyp[j - 1]:
+                dp[i][j] = dp[i - 1][j - 1]
+            else:
+                dp[i][j] = 1 + min(dp[i - 1][j], dp[i][j - 1], dp[i - 1][j - 1])
+                
+    return dp[m][n]
+
+def calculate_cer(references, hypotheses):
+    """
+    Character Error Rate (CER) = Total Character Edits / Total Reference Characters
+    """
+    total_dist = 0
+    total_len = 0
+    for ref, hyp in zip(references, hypotheses):
+        total_dist += levenshtein_distance(list(ref), list(hyp))
+        total_len += max(1, len(ref))
+    return (total_dist / total_len) if total_len > 0 else 0.0
+
+def calculate_wer(references, hypotheses):
+    """
+    Word Error Rate (WER) = Total Word Edits / Total Reference Words
+    """
+    total_dist = 0
+    total_len = 0
+    for ref, hyp in zip(references, hypotheses):
+        ref_words = ref.strip().split()
+        hyp_words = hyp.strip().split()
+        total_dist += levenshtein_distance(ref_words, hyp_words)
+        total_len += max(1, len(ref_words))
+    return (total_dist / total_len) if total_len > 0 else 0.0
+
+if __name__ == "__main__":
+    refs = ["Cộng hòa Xã hội", "Nguyễn Văn An"]
+    hyps = ["Cộng hoà Xã hội", "Nguyễn Văn An"]
+    print(f"CER: {calculate_cer(refs, hyps)*100:.2f}%")
+    print(f"WER: {calculate_wer(refs, hyps)*100:.2f}%")