rbx-proofreader 1.0.1__py3-none-any.whl → 1.1.1__py3-none-any.whl

proofreader/train/emulator/generator.py

@@ -3,6 +3,7 @@ import random
 import concurrent.futures
 import sys
 import traceback
+import multiprocessing
 from pathlib import Path
 from playwright.sync_api import sync_playwright
 from tqdm import tqdm
@@ -24,163 +25,210 @@ from proofreader.core.config import (
 
 GENERATOR_CONFIG = AUGMENTER_CONFIG["generator"]
 
-def worker_task(task_id, db, backgrounds_count):
+def process_batch(batch_ids, db, backgrounds_count, progress_counter):
     try:
-        split = "train" if random.random() < GENERATOR_CONFIG["train_split_fraction"] else "val"
-        output_name = f"trade_{task_id:05d}"
-
-        img_dir = DATASET_ROOT / split / "images"
-        lbl_dir = DATASET_ROOT / split / "labels"
-        img_dir.mkdir(parents=True, exist_ok=True)
-        lbl_dir.mkdir(parents=True, exist_ok=True)
-
-        trade_input = [[], []]
-        is_empty_trade = random.random() < GENERATOR_CONFIG["empty_trade_chance"]
-
-        if not is_empty_trade:
-            for side in [0, 1]:
-                num_items = random.randint(0, 4)
-                for _ in range(num_items):
-                    item = random.choice(db)
-                    trade_input[side].append(f"../../../../assets/thumbnails/{item['id']}.png")
-        
-        with open(AUGMENTER_PATH, 'r', encoding="utf-8") as f:
-            augmenter_js = f.read()
-
         with sync_playwright() as p:
-            browser = p.chromium.launch(headless=True)
-
-            aspect_ratio = random.uniform(GENERATOR_CONFIG["aspect_ratio_min"], GENERATOR_CONFIG["aspect_ratio_max"])
-            width = random.randint(GENERATOR_CONFIG["width_min"], GENERATOR_CONFIG["width_max"])
-            height = int(width / aspect_ratio)
-            height = max(GENERATOR_CONFIG["height_min"], min(height, GENERATOR_CONFIG["height_max"]))
+            browser = p.chromium.launch(
+                headless=True,
+                args=["--disable-gpu", "--disable-dev-shm-usage", "--no-sandbox"]
+            )
             
-            context = browser.new_context(viewport={"width": width, "height": height})
+            context = browser.new_context()
             page = context.new_page()
 
-            random_file = random.choice(TEMPLATE_FILES)
-            page.goto(f"file://{Path(random_file).absolute()}")
-
-            page.evaluate(augmenter_js, [trade_input, is_empty_trade, backgrounds_count, AUGMENTER_CONFIG])
+            with open(AUGMENTER_PATH, 'r', encoding="utf-8") as f:
+                augmenter_js = f.read()
 
-            def get_padded_yolo(element, class_id, pad_px=2):
-                box = element.bounding_box()
-                if not box: return None
-                
-                x1 = max(0, box['x'] - pad_px)
-                y1 = max(0, box['y'] - pad_px)
-                x2 = min(width, box['x'] + box['width'] + pad_px)
-                y2 = min(height, box['y'] + box['height'] + pad_px)
-                
-                new_w = x2 - x1
-                new_h = y2 - y1
-                center_x = x1 + (new_w / 2)
-                center_y = y1 + (new_h / 2)
-                
-                return [class_id, center_x / width, center_y / height, new_w / width, new_h / height]
-
-            def is_fully_visible(box, width, height, pad=4):
-                return (box['x'] - pad >= 0 and 
-                        box['y'] - pad >= 0 and 
-                        (box['x'] + box['width'] + pad) <= width and 
-                        (box['y'] + box['height'] + pad) <= height)
-            
-            label_data = []
-
-            items = page.query_selector_all("div[trade-item-card]")
-            for item in items:
-                box = item.bounding_box()
-                if box and is_fully_visible(box, width, height):
-                    card_box = get_padded_yolo(item, 0, pad_px=4)
-                    if card_box: label_data.append(card_box)
-
-                    thumb = item.query_selector(".item-card-thumb-container") 
-                    if thumb:
-                        thumb_box = get_padded_yolo(thumb, 1, pad_px=4)
-                        if thumb_box: label_data.append(thumb_box)
-                    
-                    name = item.query_selector(".item-card-name")
-                    if name:
-                        name_box = get_padded_yolo(name, 2, pad_px=4)
-                        if name_box: label_data.append(name_box)
-            
-            robux_sections = page.query_selector_all(".robux-line:not(.total-value)")
-            for section in robux_sections:
-                box = section.bounding_box()
-                if box and is_fully_visible(box, width, height, 8) and section.is_visible():
-                    line_box = get_padded_yolo(section, 3, pad_px=8)
-                    if line_box: label_data.append(line_box)
-
-                    value_element = section.query_selector(".robux-line-value") 
-                    if value_element:
-                        value_box = get_padded_yolo(value_element, 4, pad_px=4)
-                        if value_box: label_data.append(value_box)
-            
-            img_path = img_dir / f"{output_name}.png"
-            page.screenshot(path=str(img_path))
-
-            if random.random() < 0.60:
-                img = cv2.imread(str(img_path))
-                if img is not None:
-                    if random.random() < 0.5:
-                        quality = random.randint(60, 90) 
-                        _, encimg = cv2.imencode('.jpg', img, [int(cv2.IMWRITE_JPEG_QUALITY), quality])
-                        img = cv2.imdecode(encimg, 1)
-                    
-                    if random.random() < 0.4:
-                        alpha = random.uniform(0.8, 1.2)
-                        beta = random.randint(-20, 20)
-                        img = cv2.convertScaleAbs(img, alpha=alpha, beta=beta)
-                    
-                    level = random.uniform(0.5, 2.5) 
-                    noise = np.random.normal(0, level, img.shape).astype('float32')
-                    img = np.clip(img.astype('float32') + noise, 0, 255).astype('uint8')
-                    cv2.imwrite(str(img_path), img)
-            
-            label_path = lbl_dir / f"{output_name}.txt"
-            with open(label_path, "w") as f:
-                for label in label_data:
-                    f.write(f"{label[0]} {label[1]:.6f} {label[2]:.6f} {label[3]:.6f} {label[4]:.6f}\n")
+            for task_id in batch_ids:
+                generate_single_image(page, task_id, db, backgrounds_count, augmenter_js)
+                progress_counter.value += 1
 
             browser.close()
-
     except Exception:
-        print(f"Error generating task {task_id}:")
+        print(f"Batch failed starting at {batch_ids[0]}:")
         traceback.print_exc()
 
-def run_mass_generation(total_images=GENERATOR_CONFIG["total_images"], max_workers=GENERATOR_CONFIG["max_workers"]):
-    bg_files = [f for f in BACKGROUNDS_DIR.iterdir() if f.is_file() and f.name != ".gitkeep"]
-    if not bg_files:
-        print(f"❌ ERROR: No background images found in {BACKGROUNDS_DIR}")
-        print("Please add background images (JPG/PNG) to the folder before running.")
-        return
+def generate_single_image(page, task_id, db, backgrounds_count, augmenter_js):
+    split = "train" if random.random() < GENERATOR_CONFIG["train_split_fraction"] else "val"
+    output_name = f"trade_{task_id:05d}"
+    img_dir = DATASET_ROOT / split / "images"
+    lbl_dir = DATASET_ROOT / split / "labels"
+    img_dir.mkdir(parents=True, exist_ok=True)
+    lbl_dir.mkdir(parents=True, exist_ok=True)
+
+    trade_input = [[], []]
+    is_empty_trade = random.random() < GENERATOR_CONFIG["empty_trade_chance"]
+    if not is_empty_trade:
+        for side in [0, 1]:
+            num_items = random.randint(0, 4)
+            for _ in range(num_items):
+                item = random.choice(db)
+                trade_input[side].append(f"../../../../assets/thumbnails/{item['id']}.png")
     
-    valid_templates = [
-        t for t in TEMPLATE_FILES 
-        if Path(t).exists() and Path(t).name != ".gitkeep"
-    ]
-    if not valid_templates:
-        print(f"❌ ERROR: No valid HTML templates found. Checked: {TEMPLATE_FILES}")
-        print("Ensure your template files exist and are not just .gitkeep placeholders.")
-        return
+    aspect_ratio = random.uniform(GENERATOR_CONFIG["aspect_ratio_min"], GENERATOR_CONFIG["aspect_ratio_max"])
+    width = random.randint(GENERATOR_CONFIG["width_min"], GENERATOR_CONFIG["width_max"])
+    height = int(width / aspect_ratio)
+    height = max(GENERATOR_CONFIG["height_min"], min(height, GENERATOR_CONFIG["height_max"]))
     
-    if not DB_PATH.exists():
-        print(f"❌ ERROR: Item database missing at {DB_PATH}")
-        return
+    page.set_viewport_size({"width": width, "height": height})
+    random_file = random.choice(TEMPLATE_FILES)
+    page.goto(f"file://{Path(random_file).absolute()}")
     
+    zoom_factor = random.uniform(0.5, 2.0)
+    page.evaluate(f"document.body.style.zoom = '{zoom_factor}'")
+    page.evaluate(augmenter_js, [trade_input, is_empty_trade, backgrounds_count, AUGMENTER_CONFIG])
+
+    page.evaluate("""
+        async () => {
+            const imgs = Array.from(document.querySelectorAll('img'));
+            const promises = imgs.map(img => {
+                if (img.complete) return Promise.resolve();
+                return new Promise((resolve, reject) => {
+                    img.onload = resolve;
+                    img.onerror = resolve; // Continue even if image fails
+                });
+            });
+            await Promise.all(promises);
+            
+            // Final safety: Wait for the browser to paint
+            await new Promise(r => requestAnimationFrame(r));
+        }
+    """)
+
+    def get_padded_yolo(element, class_id, pad_px=2):
+        box = element.bounding_box()
+        if not box: return None
+        x1, y1 = max(0, box['x'] - pad_px), max(0, box['y'] - pad_px)
+        x2, y2 = min(width, box['x'] + box['width'] + pad_px), min(height, box['y'] + box['height'] + pad_px)
+        nw, nh = x2 - x1, y2 - y1
+        return [class_id, (x1 + nw/2)/width, (y1 + nh/2)/height, nw/width, nh/height]
+
+    def is_fully_visible(box, width, height, pad=4):
+        return (box['x'] - pad >= 0 and 
+                box['y'] - pad >= 0 and 
+                (box['x'] + box['width'] + pad) <= width and 
+                (box['y'] + box['height'] + pad) <= height)
+
+    label_data = []
+
+    items = page.query_selector_all("div[trade-item-card]")
+    for item in items:
+        box = item.bounding_box()
+        if box and is_fully_visible(box, width, height):
+            card_box = get_padded_yolo(item, 0, pad_px=4)
+            if card_box: label_data.append(card_box)
+
+            thumb = item.query_selector(".item-card-thumb-container") 
+            if thumb:
+                thumb_box = get_padded_yolo(thumb, 1, pad_px=4)
+                if thumb_box: label_data.append(thumb_box)
+
+            name = item.query_selector(".item-card-name")
+            if name:
+                name_box = get_padded_yolo(name, 2, pad_px=4)
+                if name_box: label_data.append(name_box)
+
+    robux_sections = page.query_selector_all(".robux-line:not(.total-value)")
+    for section in robux_sections:
+        box = section.bounding_box()
+        if box and is_fully_visible(box, width, height, 8) and section.is_visible():
+            line_box = get_padded_yolo(section, 3, pad_px=8)
+            if line_box: label_data.append(line_box)
+
+            value_element = section.query_selector(".robux-line-value") 
+            if value_element:
+                value_box = get_padded_yolo(value_element, 4, pad_px=4)
+                if value_box: label_data.append(value_box)
+    
+    img_buffer = page.screenshot(type="jpeg", quality=100)
+    nparr = np.frombuffer(img_buffer, np.uint8)
+    full_img = cv2.imdecode(nparr, cv2.IMREAD_COLOR)
+
+    item_cards = page.query_selector_all("div[trade-item-card]")
+    for i, card in enumerate(item_cards):
+        if not (card.is_visible() and float(card.evaluate("el => getComputedStyle(el).opacity")) > 0):
+            continue
+        thumb_container = card.query_selector(".item-card-thumb-container")
+        if thumb_container and thumb_container.is_visible():
+            img_src = thumb_container.query_selector("img").get_attribute("src")
+            item_id = Path(img_src).stem
+            box = thumb_container.bounding_box()
+            if box:
+                pad = 4
+                max_offset = 5
+                off_x = random.randint(-max_offset, max_offset)
+                off_y = random.randint(-max_offset, max_offset)
+
+                x1, y1 = int(box['x'] - pad + off_x), int(box['y'] - pad + off_y)
+                x2, y2 = int(box['x'] + box['width'] + pad + off_x), int(box['y'] + box['height'] + pad + off_y)
+                if 0 <= x1 and 0 <= y1 and x2 <= width and y2 <= height:
+                    crop = full_img[y1:y2, x1:x2]
+                    if crop.size > 0:
+                        class_dir = DATASET_ROOT / "classification" / item_id
+                        class_dir.mkdir(parents=True, exist_ok=True)
+                        if random.random() < 0.3:
+                            brightness = random.uniform(0.7, 1.3)
+                            crop = cv2.convertScaleAbs(crop, alpha=brightness, beta=0)
+                        
+                        if random.random() < 0.2:
+                            k_size = random.choice([3, 5])
+                            crop = cv2.GaussianBlur(crop, (k_size, k_size), 0)
+                        
+                        q = random.randint(70, 95)
+                        cv2.imwrite(str(class_dir / f"{output_name}_{i}.jpg"), crop, [int(cv2.IMWRITE_JPEG_QUALITY), q])
+    
+    if random.random() < 0.60:
+        if random.random() < 0.5:
+            q = random.randint(60, 90)
+            _, enc = cv2.imencode('.jpg', full_img, [int(cv2.IMWRITE_JPEG_QUALITY), q])
+            full_img = cv2.imdecode(enc, 1)
+
+        if random.random() < 0.4:
+            full_img = cv2.convertScaleAbs(full_img, alpha=random.uniform(0.8, 1.2), beta=random.randint(-20, 20))
+        
+        noise = np.random.normal(0, random.uniform(0.5, 2.5), full_img.shape).astype('float32')
+        full_img = np.clip(full_img.astype('float32') + noise, 0, 255).astype('uint8')
+    
+    cv2.imwrite(str(img_dir / f"{output_name}.jpg"), full_img, [int(cv2.IMWRITE_JPEG_QUALITY), 95])
+    
+    with open(lbl_dir / f"{output_name}.txt", "w") as f:
+        for label in label_data:
+            f.write(f"{label[0]} {label[1]:.6f} {label[2]:.6f} {label[3]:.6f} {label[4]:.6f}\n")
+
+def run_mass_generation(total_images=65536, max_workers=24):
     with open(DB_PATH, "r") as f:
         db = json.load(f)
     
-    backgrounds_count = len([f for f in BACKGROUNDS_DIR.iterdir() if f.is_file()]) - 1
-
     setup_dataset_directories(force_reset=True)
 
-    print(f"Starting generation of {total_images} images using {max_workers} processes...")
+    batch_size = 500
+    all_ids = list(range(total_images))
+    chunks = [all_ids[i:i + batch_size] for i in range(0, len(all_ids), batch_size)]
+
+    backgrounds_count = len([f for f in BACKGROUNDS_DIR.iterdir() if f.is_file()])
+
+    manager = multiprocessing.Manager()
+    progress_counter = manager.Value('i', 0)
+
+    print(f"Generating {total_images} images using {max_workers} workers...")
 
     with concurrent.futures.ProcessPoolExecutor(max_workers=max_workers) as executor:
-        futures = [executor.submit(worker_task, i, db, backgrounds_count) for i in range(total_images)]
-        for _ in tqdm(concurrent.futures.as_completed(futures), total=total_images):
-            pass
+        futures = [
+            executor.submit(process_batch, chunk, db, backgrounds_count, progress_counter) 
+            for chunk in chunks
+        ]
+
+        with tqdm(total=total_images, desc="Generating Images") as pbar:
+            last_val = 0
+            while True:
+                done, not_done = concurrent.futures.wait(futures, timeout=0.5)
+
+                current_val = progress_counter.value
+                pbar.update(current_val - last_val)
+                last_val = current_val
+                
+                if len(not_done) == 0:
+                    break
 
 if __name__ == "__main__":
-    run_mass_generation(total_images=16, max_workers=8)
+    run_mass_generation()

proofreader/train/{train.py → yolo_trainer.py}

@@ -1,7 +1,7 @@
 from ultralytics import YOLO
 from ..core.config import TRAINING_CONFIG, DATA_YAML_PATH
 
-def train_model(device):
+def train_yolo(device):
     model = YOLO("yolo11n.pt")
 
     model.train(
@@ -27,7 +27,7 @@ def train_model(device):
         workers = 8
     )
 
-def finish_training(file_path):
+def finish_training(file_path, device):
     model = YOLO(file_path)
 
     model.train(
@@ -35,10 +35,7 @@ def finish_training(file_path):
         epochs = 32,
         close_mosaic = 32,
         patience = 20,
-        imgsz = 640,
-        batch = 24,
-        device = 0 # Change to "cpu" if no CUDA devices
+        imgsz = TRAINING_CONFIG["img_size"],
+        batch = TRAINING_CONFIG["batch_size"],
+        device = device
     )
-
-if __name__ == "__main__":
-    train_model()

rbx_proofreader-1.1.1.dist-info/METADATA

@@ -0,0 +1,160 @@
+Metadata-Version: 2.4
+Name: rbx-proofreader
+Version: 1.1.1
+Summary: Visual trade detection and OCR engine
+License: MIT
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.12
+Requires-Python: >=3.12
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: easyocr>=1.7.0
+Requires-Dist: numpy>=1.24.0
+Requires-Dist: opencv-python>=4.8.0
+Requires-Dist: Pillow>=10.0.0
+Requires-Dist: rapidfuzz>=3.0.0
+Requires-Dist: requests>=2.31.0
+Requires-Dist: torch>=2.0.0
+Requires-Dist: tqdm>=4.66.0
+Requires-Dist: transformers>=4.30.0
+Requires-Dist: ultralytics>=8.0.0
+Provides-Extra: train
+Requires-Dist: playwright>=1.40.0; extra == "train"
+Dynamic: license-file
+
+# Proofreader 🔍
+
+A high-speed vision pipeline for reading Roblox trade screenshots.
+
+[![PyPI](https://img.shields.io/pypi/v/rbx-proofreader?color=blue&label=PyPI)](https://pypi.org/project/rbx-proofreader/)
+[![Downloads](https://static.pepy.tech/badge/rbx-proofreader)](https://pepy.tech/project/rbx-proofreader)
+[![Python](https://img.shields.io/pypi/pyversions/rbx-proofreader?logo=python&logoColor=white&color=blue)](https://pypi.org/project/rbx-proofreader/)
+[![License](https://img.shields.io/badge/license-MIT-blue)](LICENSE)
+[![Build Status](https://github.com/lucacrose/proofreader/actions/workflows/build.yml/badge.svg)](https://github.com/lucacrose/proofreader/actions)
+[![GPU](https://img.shields.io/badge/GPU-CUDA-blueviolet)](https://developer.nvidia.com/cuda-zone)
+[![YOLOv11](https://img.shields.io/badge/model-YOLOv11-blueviolet)](https://github.com/ultralytics/ultralytics)
+
+Proofreader transforms unstructured screenshots of Roblox trades ("proofs", hence "proofreader") into structured Python dictionaries. By combining **YOLOv11** for object detection, **CLIP** for visual similarity, and **EasyOCR**, it achieves high accuracy across diverse UI themes, resolutions, and extensions.
+
+## Why Proofreader?
+
+Roblox trade screenshots are commonly used as proof in marketplaces, moderation workflows, and value analysis, yet they are manually verified and error-prone. Proofreader automates this process by converting screenshots into structured, verifiable data in milliseconds.
+
+
+## Example
+
+![Example](https://github.com/lucacrose/proofreader/raw/main/docs/assets/example.png)
+
+## ⚡ Performance
+
+Tested on an **RTX 5070** using $n=500$ real-world "worst-case" user screenshots (compressed, cropped, and varied UI).
+
+| Metric                  | Result (E2E)                |
+|:------------------------|:----------------------------|
+| Exact Match Accuracy    | 97.2% (95% CI: 95.4–98.5%)  |
+| Median latency          | 36.8 ms                     |
+| 95th percentile latency | 73.4 ms                     |
+
+> [!NOTE]
+> End-to-End **(E2E)** latency includes image loading, YOLO detection, spatial organization, CLIP similarity matching, and OCR fallback.
+
+## ✨ Key Features
+
+- **Sub-40ms Latency:** Optimized with "Fast-Path" logic that skips OCR for high-confidence visual matches, ensuring near-instant processing.
+
+- **Multi-modal decision engine:** Weighs visual embeddings against OCR text to resolve identities across 2,500+ distinct item classes.
+
+- **Fuzzy Logic Recovery:** Built-in string distance matching corrects OCR typos and text obscurations against a local asset database.
+
+- **Theme & Scale Agnostic:** Robust performance across various UI themes (Dark/Light), resolutions, and custom display scales.
+
+## 💻 Quick Start
+
+### Installation
+
+```bash
+pip install rbx-proofreader
+```
+
+> [!IMPORTANT]
+> **Hardware Acceleration:** Proofreader automatically detects NVIDIA GPUs. For sub-40ms performance, ensure you have the CUDA-enabled version of PyTorch installed. If a CPU-only environment is detected on a GPU-capable machine, the engine will provide the exact `pip` command to fix your environment.
+
+### Usage
+
+```py
+import proofreader
+
+# Extract metadata from a screenshot
+data = proofreader.get_trade_data("trade_proof.png")
+
+print(f"Items Out: {data['outgoing']['item_count']}")
+print(f"Robux In: {data['incoming']['robux_value']}")
+```
+
+> [!TIP]
+> **First Run:** On your first execution, Proofreader will automatically download the model weights and item database (~360MB). Subsequent runs will use the local cache for maximum speed.
+
+## 🧩 How it Works
+The model handles the inconsistencies of user-generated screenshots (varied crops, UI themes, and extensions) through a multi-stage process:
+
+1. **Detection:** YOLOv11 localizes item cards, thumbnails, and robux containers.
+
+2. **Spatial Organization:** Assigns child elements (names/values) to parents and determines trade side.
+
+3. **Identification:** CLIP performs similarity matching. High-confidence results become Resolved Items immediately.
+
+4. **Heuristic Judge:** Low-confidence visual matches trigger OCR and fuzzy-logic reconciliation.
+
+![Diagram](https://github.com/lucacrose/proofreader/raw/main/docs/assets/flow_diagram.png)
+
+## 📊 Data Schema
+The `get_trade_data()` function returns a structured dictionary containing `incoming` and `outgoing` trade sides.
+
+| Key | Type | Description |
+| :--- | :--- | :--- |
+| `item_count` | `int` | Number of distinct item boxes detected. |
+| `robux_value` | `int` | Total Robux parsed from the trade. |
+| `items` | `list` | List of `ResolvedItem` objects containing `id` and `name`. |
+
+**ResolvedItem Schema:**
+
+| Property | Type | Description |
+| :--- | :--- | :--- |
+| `id` | `int` | The official Roblox Asset ID. |
+| `name` | `str` | Canonical item name from the database. |
+
+## 🏗️ Development & Training
+To set up a custom training environment for the YOLO and CLIP models:
+
+```bash
+# 1. Clone and Install
+git clone https://github.com/lucacrose/proofreader.git
+cd proofreader
+pip install -e ".[train]"
+
+# 2. Initialize Database
+python scripts/setup_items.py
+
+# 3. Training
+# Place backgrounds in src/proofreader/train/emulator/backgrounds
+# Place HTML templates in src/proofreader/train/emulator/templates
+python scripts/train_models.py
+```
+
+> [!CAUTION]
+> **GPU Required:** Training is not recommended on a CPU. Final models save to `runs/train/weights/best.pt`. Rename to `yolo.pt` and move to `src/assets/weights`.
+
+## 🛠️ Tech Stack
+
+- **Vision:** YOLOv11 (Detection), CLIP (Embeddings), OpenCV (Processing)
+- **OCR:** EasyOCR
+- **Logic:** RapidFuzz (Fuzzy String Matching)
+- **Core:** Python 3.12, PyTorch, NumPy
+
+## 🤝 Contributing
+
+Contributions are welcome! Please open an issue or submit a pull request.
+
+## 📜 License
+
+This project is licensed under the MIT License.

rbx_proofreader-1.1.1.dist-info/RECORD

@@ -0,0 +1,17 @@
+proofreader/__init__.py,sha256=YVsRxmHmC2nvCrxvNmZX230B1s5k36RFM51kElXSxB4,285
+proofreader/main.py,sha256=01G_-ppevuNNafi-QCc6UB_Y2NuIW6sDoZwvjjdm1B0,5220
+proofreader/core/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+proofreader/core/config.py,sha256=8L6MTBn3Z3Xa0bjPYt5q-OI-mm0-wMqeDSS0beAQ1fk,5906
+proofreader/core/detector.py,sha256=em2Kx0v96Zofi4kK5ipWlqMX9czq9YobHuEGuZkAQEc,987
+proofreader/core/matcher.py,sha256=4URgBb6EgBaCNFpafjnQrIot9KeIwoYUUNraaC9nlIk,3603
+proofreader/core/ocr.py,sha256=FFhIS1TVrqSXUPGOll5RNbHX18q7de4xFUP1ewrnhSc,3652
+proofreader/core/resolver.py,sha256=DTbf5qyQaJrBbw1QWQQJ_BZf_dg003p_xH8RMpI6sn8,2685
+proofreader/core/schema.py,sha256=ga_7cYCBO13yFvLAtyAgDw7CFEb9c8Ui85SJDu2pcsA,2512
+proofreader/train/clip_trainer.py,sha256=6hiVrJ6WX6m13E3FE8kouIxXjQo3GPrU_8X266oeXqs,6416
+proofreader/train/yolo_trainer.py,sha256=nOHPrYmBuefsUyiGEYqboNU6i3pykBXE0U4HYwNaqg8,986
+proofreader/train/emulator/generator.py,sha256=_l7qFLSoQxPYUKLDrqVIS-0sUs5FkjBK7ENWmZ-q2ls,9681
+rbx_proofreader-1.1.1.dist-info/licenses/LICENSE,sha256=eHSaONn9P_ZcYiY9QCi_XzVARIoQu7l2AI5BtFGA_BY,1069
+rbx_proofreader-1.1.1.dist-info/METADATA,sha256=CNi-FAGJvwoEL6LPmpL8m39canCcXsg_idlPTQFVeFA,6568
+rbx_proofreader-1.1.1.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
+rbx_proofreader-1.1.1.dist-info/top_level.txt,sha256=U3s8IVdLtGeGD3JgMmCHUgAsFhZXSSamp3vIojAFTxU,12
+rbx_proofreader-1.1.1.dist-info/RECORD,,

proofreader/train/builder.py

@@ -1,94 +0,0 @@
-import os
-import json
-import torch
-from PIL import Image
-from ..core.config import DB_PATH, CACHE_PATH, THUMBNAILS_DIR, BUILDER_BATCH_SIZE
-
-class EmbeddingBuilder:
-    def __init__(self, model, processor):
-        self.model = model
-        self.processor = processor
-
-    def get_clip_embedding(self, pil_img):
-        inputs = self.processor(images=pil_img, return_tensors="pt", padding=True).to(self.model.device)
-        with torch.no_grad():
-            features = self.model.get_image_features(**inputs)
-        return features.cpu().numpy().flatten()
-
-    def build(self, batch_size=BUILDER_BATCH_SIZE):
-        self.model.eval()
-        print(f"Starting build process...")
-        print(f"Source Images: {THUMBNAILS_DIR}")
-        print(f"Item Database: {DB_PATH}")
-        
-        if not os.path.exists(DB_PATH):
-            print(f"Error: Missing {DB_PATH}. Cannot map IDs to Names.")
-            return
-            
-        with open(DB_PATH, "r") as f:
-            items = json.load(f)
-        
-        embedding_bank = {}
-        item_names = []
-        
-        if not os.path.exists(THUMBNAILS_DIR):
-            print(f"Error: Image directory {THUMBNAILS_DIR} not found.")
-            return
-
-        image_files = [f for f in os.listdir(THUMBNAILS_DIR) if f.lower().endswith(('.png', '.jpg', '.jpeg'))]
-        total_files = len(image_files)
-        
-        embedding_bank = {}
-        item_names = []
-
-        for i in range(0, total_files, batch_size):
-            batch_files = image_files[i : i + batch_size]
-            batch_imgs = []
-            batch_item_names = []
-
-            for filename in batch_files:
-                item_id = os.path.splitext(filename)[0]
-                item_info = next((item for item in items if str(item.get("id")) == item_id), None)
-                
-                if item_info:
-                    try:
-                        img_path = os.path.join(THUMBNAILS_DIR, filename)
-                        raw_img = Image.open(img_path)
-
-                        if raw_img.mode in ("RGBA", "P"):
-                            bg = Image.new("RGB", raw_img.size, (255, 255, 255))
-                            bg.paste(raw_img.convert("RGBA"), (0, 0), raw_img.convert("RGBA"))
-                            img = bg
-                        else:
-                            img = raw_img.convert("RGB")
-                            
-                        batch_imgs.append(img)
-                        batch_item_names.append(item_info["name"])
-                    except Exception as e:
-                        print(f"Could not load {filename}: {e}")
-
-            if not batch_imgs:
-                continue
-            try:
-                inputs = self.processor(images=batch_imgs, return_tensors="pt", padding=True).to(self.model.device)
-                with torch.no_grad():
-                    features = self.model.get_image_features(**inputs)
-                
-                features_numpy = features.cpu().numpy()
-                for name, emb in zip(batch_item_names, features_numpy):
-                    embedding_bank[name] = emb
-                    item_names.append(name)
-                
-                print(f"Progress: {min(i + batch_size, total_files)}/{total_files} items indexed...")
-            except Exception as e:
-                print(f"Batch processing error: {e}")
-        
-        output_data = {
-            'embeddings': embedding_bank, 
-            'names': item_names
-        }
-        
-        torch.save(output_data, CACHE_PATH)
-        print(f"\n✅ Build Complete!")
-        print(f"Target: {CACHE_PATH}")
-        print(f"Total Embeddings Saved: {len(embedding_bank)}")