GameSentenceMiner 2.14.9__py3-none-any.whl → 2.14.11__py3-none-any.whl

GameSentenceMiner/util/get_overlay_coords.py

@@ -0,0 +1,366 @@
+import asyncio
+import io
+import base64
+import math
+import os
+import time
+from PIL import Image
+from typing import Dict, Any, List, Tuple
+
+# Local application imports
+from GameSentenceMiner.ocr.gsm_ocr_config import set_dpi_awareness
+from GameSentenceMiner.util.configuration import get_config, is_windows
+from GameSentenceMiner.util.electron_config import get_ocr_language
+from GameSentenceMiner.obs import get_screenshot_PIL, logger
+from GameSentenceMiner.web.texthooking_page import send_word_coordinates_to_overlay
+
+# Conditionally import OCR engines
+try:
+    if os.path.exists(os.path.expanduser('~/.config/oneocr/oneocr.dll')):
+        from GameSentenceMiner.owocr.owocr.ocr import OneOCR
+    else:
+        OneOCR = None
+    from GameSentenceMiner.owocr.owocr.ocr import GoogleLens, get_regex
+except ImportError as import_err:
+    GoogleLens, OneOCR, get_regex = None, None, None
+except Exception as e:
+    GoogleLens, OneOCR, get_regex = None, None, None
+    logger.error(f"Error importing OCR engines: {e}", exc_info=True)
+
+# Conditionally import screenshot library
+try:
+    import mss
+except ImportError:
+    mss = None
+
+class OverlayProcessor:
+    """
+    Handles the entire overlay process from screen capture to text extraction.
+
+    This class encapsulates the logic for taking screenshots, identifying text
+    regions, performing OCR, and processing the results into a structured format
+    with pixel coordinates.
+    """
+
+    def __init__(self):
+        """Initializes the OCR engines and configuration."""
+        self.config = get_config()
+        self.oneocr = None
+        self.lens = None
+        self.regex = None
+        self.ready = False
+
+        try:
+            if self.config.overlay.websocket_port and all([GoogleLens, get_regex]):
+                logger.info("Initializing OCR engines...")
+                if OneOCR:
+                    self.oneocr = OneOCR(lang=get_ocr_language())
+                self.lens = GoogleLens(lang=get_ocr_language())
+                self.ocr_language = get_ocr_language()
+                self.regex = get_regex(self.ocr_language)
+                logger.info("OCR engines initialized.")
+                self.ready = True
+            else:
+                logger.warning("OCR dependencies not found or websocket port not configured. OCR functionality will be disabled.")
+            
+            if is_windows:
+                set_dpi_awareness()
+                
+            if not mss:
+                logger.warning("MSS library not found. Screenshot functionality may be limited.")
+        except Exception as e:
+            logger.error(f"Error initializing OCR engines for overlay, try installing owocr in OCR tab of GSM: {e}", exc_info=True)
+            self.oneocr = None
+            self.lens = None
+            self.regex = None
+            
+    async def find_box_and_send_to_overlay(self, sentence_to_check: str = None):
+        """
+        Sends the detected text boxes to the overlay via WebSocket.
+        """
+        boxes = await self.find_box_for_sentence(sentence_to_check)
+        logger.info(f"Sending {len(boxes)} boxes to overlay.")
+        await send_word_coordinates_to_overlay(boxes)
+
+    async def find_box_for_sentence(self, sentence_to_check: str = None) -> List[Dict[str, Any]]:
+        """
+        Public method to perform OCR and find text boxes for a given sentence.
+        
+        This is a wrapper around the main work-horse method, providing
+        error handling.
+        """
+        try:
+            return await self._do_work(sentence_to_check)
+        except Exception as e:
+            logger.error(f"Error during OCR processing: {e}", exc_info=True)
+            return []
+
+    def _get_full_screenshot(self) -> Tuple[Image.Image | None, int, int]:
+        """Captures a screenshot of the configured monitor."""
+        if not mss:
+            raise RuntimeError("MSS screenshot library is not installed.")
+        
+        with mss.mss() as sct:
+            monitors = sct.monitors
+            # Index 0 is the 'all monitors' virtual screen, so we skip it.
+            monitor_list = monitors[1:] if len(monitors) > 1 else [monitors[0]]
+            
+            monitor_index = self.config.overlay.monitor_to_capture
+            if monitor_index >= len(monitor_list):
+                logger.error(f"Monitor index {monitor_index} is out of bounds. Found {len(monitor_list)} monitors.")
+                return None, 0, 0
+                
+            monitor = monitor_list[monitor_index]
+            sct_img = sct.grab(monitor)
+            img = Image.frombytes('RGB', sct_img.size, sct_img.bgra, 'raw', 'BGRX')
+            
+            return img, monitor['width'], monitor['height']
+
+    def _create_composite_image(
+        self, 
+        full_screenshot: Image.Image,
+        crop_coords_list: List[Tuple[int, int, int, int]],
+        monitor_width: int,
+        monitor_height: int
+    ) -> Image.Image:
+        """
+        Creates a new image by pasting cropped text regions onto a transparent background.
+        This isolates text for more accurate secondary OCR.
+        """
+        if not crop_coords_list:
+            return full_screenshot
+
+        # Create a transparent canvas
+        composite_img = Image.new("RGBA", (monitor_width, monitor_height), (0, 0, 0, 0))
+
+        for crop_coords in crop_coords_list:
+            # Ensure crop coordinates are within image bounds
+            x1, y1, x2, y2 = crop_coords
+            x1 = max(0, min(x1, full_screenshot.width))
+            y1 = max(0, min(y1, full_screenshot.height))
+            x2 = max(x1, min(x2, full_screenshot.width))
+            y2 = max(y1, min(y2, full_screenshot.height))
+            
+            # Skip if the coordinates result in an invalid box
+            if x1 >= x2 or y1 >= y2:
+                continue
+            try:
+                cropped_image = full_screenshot.crop((x1, y1, x2, y2))
+            except ValueError:
+                logger.warning("Error cropping image, using original image")
+                return full_screenshot
+            # Paste the cropped image onto the canvas at its original location
+            paste_x = math.floor(x1)
+            paste_y = math.floor(y1)
+            composite_img.paste(cropped_image, (paste_x, paste_y))
+        
+        return composite_img
+
+    async def _do_work(self, sentence_to_check: str = None) -> Tuple[List[Dict[str, Any]], int]:
+        """The main OCR workflow."""
+        if not self.lens:
+            logger.error("OCR engines are not initialized. Cannot perform OCR for Overlay.")
+            return []
+
+        # 1. Get screenshot
+        full_screenshot, monitor_width, monitor_height = self._get_full_screenshot()
+        if not full_screenshot:
+            logger.warning("Failed to get a screenshot.")
+            return []
+        if self.oneocr:
+            # 2. Use OneOCR to find general text areas (fast)
+            _, _, oneocr_results, crop_coords_list = self.oneocr(
+                full_screenshot,
+                return_coords=True,
+                multiple_crop_coords=True,
+                return_one_box=False,
+                furigana_filter_sensitivity=None # Disable furigana filtering
+            )
+
+            # 3. Create a composite image with only the detected text regions
+            composite_image = self._create_composite_image(
+                full_screenshot, 
+                crop_coords_list, 
+                monitor_width, 
+                monitor_height
+            )
+        else:
+            composite_image = full_screenshot
+        
+        # 4. Use Google Lens on the cleaner composite image for higher accuracy
+        res = self.lens(
+            composite_image,
+            return_coords=True,
+            furigana_filter_sensitivity=None # Disable furigana filtering
+        )
+        
+        if len(res) != 3:
+            return oneocr_results
+        
+        _, _, coords = res
+
+        if not res or not coords:
+            return oneocr_results
+        
+        # 5. Process the high-accuracy results into the desired format
+        extracted_data = self._extract_text_with_pixel_boxes(
+            api_response=coords,
+            original_width=monitor_width,
+            original_height=monitor_height,
+            crop_x=0,
+            crop_y=0,
+            crop_width=composite_image.width,
+            crop_height=composite_image.height
+        )
+        
+        return extracted_data
+
+    def _extract_text_with_pixel_boxes(
+        self,
+        api_response: Dict[str, Any],
+        original_width: int,
+        original_height: int,
+        crop_x: int,
+        crop_y: int,
+        crop_width: int,
+        crop_height: int
+    ) -> List[Dict[str, Any]]:
+        """
+        Parses Google Lens API response and converts normalized coordinates
+        to absolute pixel coordinates.
+        """
+        results = []
+        try:
+            paragraphs = api_response["objects_response"]["text"]["text_layout"]["paragraphs"]
+        except (KeyError, TypeError):
+            return []  # Return empty if the expected structure isn't present
+
+        for para in paragraphs:
+            for line in para.get("lines", []):
+                line_text_parts = []
+                word_list = []
+
+                for word in line.get("words", []):
+                    word_text = word.get("plain_text", "")
+                    line_text_parts.append(word_text)
+                    
+                    word_box = self._convert_box_to_pixels_v2(
+                        word["geometry"]["bounding_box"],
+                        crop_x, crop_y, crop_width, crop_height
+                    )
+                    
+                    word_list.append({
+                        "text": word_text,
+                        "bounding_rect": word_box
+                    })
+                
+                if not line_text_parts:
+                    continue
+                
+                full_line_text = "".join(line_text_parts)
+                line_box = self._convert_box_to_pixels_v2(
+                    line["geometry"]["bounding_box"],
+                    crop_x, crop_y, crop_width, crop_height
+                )
+
+                results.append({
+                    "text": full_line_text,
+                    "bounding_rect": line_box,
+                    "words": word_list
+                })
+        return results
+
+    def _convert_box_to_pixels_v2(
+        self,
+        bbox_data: Dict[str, float],
+        crop_x: int,
+        crop_y: int,
+        crop_width: int,
+        crop_height: int
+    ) -> Dict[str, float]:
+        """
+        Simplified conversion: scales normalized bbox to pixel coordinates within
+        the cropped region, then offsets by the crop position. Ignores rotation.
+        """
+        cx, cy = bbox_data['center_x'], bbox_data['center_y']
+        w, h = bbox_data['width'], bbox_data['height']
+
+        # Scale normalized coordinates to pixel coordinates relative to the crop area
+        box_width_px = w * crop_width
+        box_height_px = h * crop_height
+        
+        # Calculate center within the cropped area and then add the crop offset
+        center_x_px = (cx * crop_width) + crop_x
+        center_y_px = (cy * crop_height) + crop_y
+
+        # Calculate corners (unrotated)
+        half_w_px, half_h_px = box_width_px / 2, box_height_px / 2
+        return {
+            "x1": center_x_px - half_w_px, "y1": center_y_px - half_h_px,
+            "x2": center_x_px + half_w_px, "y2": center_y_px - half_h_px,
+            "x3": center_x_px + half_w_px, "y3": center_y_px + half_h_px,
+            "x4": center_x_px - half_w_px, "y4": center_y_px + half_h_px,
+        }
+
+async def main_test_screenshot():
+    """
+    A test function to demonstrate screenshot and image composition.
+    This is preserved from your original __main__ block.
+    """
+    processor = OverlayProcessor()
+    
+    # Use the class method to get the screenshot
+    img, monitor_width, monitor_height = processor._get_full_screenshot()
+    if not img:
+        logger.error("Could not get screenshot for test.")
+        return
+        
+    img.show()
+    
+    # Create a transparent image with the same size as the monitor
+    new_img = Image.new("RGBA", (monitor_width, monitor_height), (0, 0, 0, 0))
+    
+    # Calculate coordinates to center the captured image (if it's not full-screen)
+    left = (monitor_width - img.width) // 2
+    top = (monitor_height - img.height) // 2
+    
+    print(f"Image size: {img.size}, Monitor size: {monitor_width}x{monitor_height}")
+    print(f"Pasting at: Left={left}, Top={top}")
+    
+    new_img.paste(img, (left, top))
+    new_img.show()
+
+async def main_run_ocr():
+    """
+    Main function to demonstrate running the full OCR process.
+    """
+    processor = OverlayProcessor()
+    results, _ = await processor.find_box_for_sentence()
+    if results:
+        import json
+        print("OCR process completed successfully.")
+        # print(json.dumps(results, indent=2, ensure_ascii=False))
+        # Find first result with some text
+        for result in results:
+            if result.get("text"):
+                print(f"Found line: '{result['text']}'")
+                print(f"  - Line BBox: {result['bounding_rect']}")
+                if result.get("words"):
+                    print(f"  - First word: '{result['words'][0]['text']}' BBox: {result['words'][0]['bounding_rect']}")
+                break
+    else:
+        print("OCR process did not find any text.")
+
+
+if __name__ == '__main__':
+    try:
+        # To run the screenshot test:
+        # asyncio.run(main_test_screenshot())
+        
+        # To run the full OCR process:
+        asyncio.run(main_run_ocr())
+
+    except KeyboardInterrupt:
+        logger.info("Script terminated by user.")
+    except Exception as e:
+        logger.error(f"An error occurred in the main execution block: {e}", exc_info=True)

GameSentenceMiner/util/gsm_utils.py

@@ -0,0 +1,323 @@
+import json
+import os
+import random
+import re
+import socket
+import string
+import subprocess
+import threading
+import time
+from datetime import datetime
+from pathlib import Path
+
+import requests
+from rapidfuzz import process
+
+from GameSentenceMiner.util.configuration import logger, get_config, get_app_directory
+
+SCRIPTS_DIR = r"E:\Japanese Stuff\agent-v0.1.4-win32-x64\data\scripts"
+
+def run_new_thread(func):
+    thread = threading.Thread(target=func, daemon=True)
+    thread.start()
+    return thread
+
+def make_unique_file_name(path):
+    path = Path(path)
+    current_time = datetime.now().strftime('%Y-%m-%d-%H-%M-%S-%f')[:-3]
+    return str(path.parent / f"{path.stem}_{current_time}{path.suffix}")
+
+def make_unique(text):
+    """
+    Generate a unique string by appending a timestamp to the input text.
+    This is useful for creating unique filenames or identifiers.
+    """
+    current_time = datetime.now().strftime('%Y-%m-%d-%H-%M-%S-%f')[:-3]
+    return f"{text}_{current_time}"
+
+def sanitize_filename(filename):
+        return re.sub(r'[ <>:"/\\|?*\x00-\x1F]', '', filename)
+
+
+def get_random_digit_string():
+    return ''.join(random.choice(string.digits) for i in range(9))
+
+
+def timedelta_to_ffmpeg_friendly_format(td_obj):
+    total_seconds = td_obj.total_seconds()
+    hours, remainder = divmod(total_seconds, 3600)
+    minutes, seconds = divmod(remainder, 60)
+    return "{:02}:{:02}:{:06.3f}".format(int(hours), int(minutes), seconds)
+
+
+def get_file_modification_time(file_path):
+    mod_time_epoch = os.path.getmtime(file_path)
+    mod_time = datetime.fromtimestamp(mod_time_epoch)
+    return mod_time
+
+
+def get_process_id_by_title(game_title):
+    powershell_command = f"Get-Process | Where-Object {{$_.MainWindowTitle -like '*{game_title}*'}} | Select-Object -First 1 -ExpandProperty Id"
+    process_id = subprocess.check_output(["powershell", "-Command", powershell_command], text=True).strip()
+    logger.info(f"Process ID for {game_title}: {process_id}")
+    return process_id
+
+
+def get_script_files(directory):
+    script_files = []
+    for root, dirs, files in os.walk(directory):
+        for file in files:
+            if file.endswith(".js"):  # Assuming the scripts are .js files
+                script_files.append(os.path.join(root, file))
+    return script_files
+
+
+def filter_steam_scripts(scripts):
+    return [script for script in scripts if "PC_Steam" in os.path.basename(script)]
+
+
+def extract_game_name(script_path):
+    # Remove directory and file extension to get the name part
+    script_name = os.path.basename(script_path)
+    game_name = script_name.replace("PC_Steam_", "").replace(".js", "")
+    return game_name.replace("_", " ").replace(".", " ")
+
+
+def find_most_similar_script(game_title, steam_scripts):
+    # Create a list of game names from the script paths
+    game_names = [extract_game_name(script) for script in steam_scripts]
+
+    # Use rapidfuzz to find the closest match
+    best_match = process.extractOne(game_title, game_names)
+
+    if best_match:
+        matched_game_name, confidence_score, index = best_match
+        return steam_scripts[index], matched_game_name, confidence_score
+    return None, None, None
+
+
+def find_script_for_game(game_title):
+    script_files = get_script_files(SCRIPTS_DIR)
+
+    steam_scripts = filter_steam_scripts(script_files)
+
+    best_script, matched_game_name, confidence = find_most_similar_script(game_title, steam_scripts)
+
+
+    if best_script:
+        logger.info(f"Found Script: {best_script}")
+        return best_script
+    else:
+        logger.warning("No similar script found.")
+
+
+def run_agent_and_hook(pname, agent_script):
+    command = f'agent --script=\"{agent_script}\" --pname={pname}'
+    logger.info("Running and Hooking Agent!")
+    try:
+        dos_process = subprocess.Popen(command, shell=True)
+        dos_process.wait()  # Wait for the process to complete
+        logger.info("Agent script finished or closed.")
+    except Exception as e:
+        logger.error(f"Error occurred while running agent script: {e}")
+
+    keep_running = False
+
+
+# def run_command(command, shell=False, input=None, capture_output=False, timeout=None, check=False, **kwargs):
+#     # Use shell=True if the OS is Linux, otherwise shell=False
+#     if is_linux():
+#         return subprocess.run(command, shell=True, input=input, capture_output=capture_output, timeout=timeout,
+#                               check=check, **kwargs)
+#     else:
+#         return subprocess.run(command, shell=shell, input=input, capture_output=capture_output, timeout=timeout,
+#                               check=check, **kwargs)
+def remove_html_and_cloze_tags(text):
+    text = re.sub(r'<.*?>', '', re.sub(r'{{c\d+::(.*?)(::.*?)?}}', r'\1', text))
+    return text
+
+
+def combine_dialogue(dialogue_lines, new_lines=None):
+    if not dialogue_lines:  # Handle empty input
+        return []
+
+    if new_lines is None:
+        new_lines = []
+
+    if len(dialogue_lines) == 1 and '「' not in dialogue_lines[0]:
+        new_lines.append(dialogue_lines[0])
+        return new_lines
+
+    character_name = dialogue_lines[0].split("「")[0]
+    text = character_name + "「"
+
+    for i, line in enumerate(dialogue_lines):
+        if not line.startswith(character_name + "「"):
+            text = text + "」" + get_config().advanced.multi_line_line_break
+            new_lines.append(text)
+            new_lines.extend(combine_dialogue(dialogue_lines[i:]))
+            break
+        else:
+            text +=  (get_config().advanced.multi_line_line_break if i > 0 else "") + line.split("「")[1].rstrip("」") + ""
+    else:
+        text = text + "」"
+        new_lines.append(text)
+
+    return new_lines
+
+def wait_for_stable_file(file_path, timeout=10, check_interval=0.5):
+    elapsed_time = 0
+    last_size = -1
+
+    logger.info(f"Waiting for file '{file_path}' to stabilize or become accessible...")
+
+    while elapsed_time < timeout:
+        try:
+            current_size = os.path.getsize(file_path)
+            if current_size == last_size:
+                try:
+                    with open(file_path, 'rb'):
+                        return True
+                except IOError:
+                    pass
+            last_size = current_size
+        except FileNotFoundError:
+            last_size = -1
+        except Exception as e:
+            logger.warning(f"Error checking file {file_path}, will retry: {e}")
+            last_size = -1
+
+        time.sleep(check_interval)
+        elapsed_time += check_interval
+
+    logger.warning(f"File '{file_path}' did not stabilize or become accessible within {timeout} seconds. Continuing...")
+    return False
+
+def isascii(s: str):
+    try:
+        return s.isascii()
+    except:
+        try:
+            s.encode("ascii")
+            return True
+        except:
+            return False
+
+def do_text_replacements(text, replacements_json):
+    if not text:
+        return text
+
+    replacements = {}
+    if os.path.exists(replacements_json):
+        with open(replacements_json, 'r', encoding='utf-8') as f:
+            replacements.update(json.load(f))
+
+    if replacements.get("enabled", False):
+        orig_text = text
+        filters = replacements.get("args", {}).get("replacements", {})
+        for fil, replacement in filters.items():
+            if not fil:
+                continue
+            if fil.startswith("re:"):
+                pattern = fil[3:]
+                try:
+                    text = re.sub(pattern, replacement, text)
+                except Exception:
+                    logger.error(f"Invalid regex pattern: {pattern}")
+                    continue
+            if isascii(fil):
+                text = re.sub(r"\b{}\b".format(re.escape(fil)), replacement, text)
+            else:
+                text = text.replace(fil, replacement)
+        if text != orig_text:
+            logger.info(f"Text replaced: '{orig_text}' -> '{text}' using replacements.")
+    return text
+
+
+def open_audio_in_external(fileabspath, shell=False):
+    logger.info(f"Opening audio in external program...")
+    try:
+        if shell:
+            subprocess.Popen(f' "{get_config().audio.external_tool}" "{fileabspath}" ', shell=True)
+        else:
+            subprocess.Popen([get_config().audio.external_tool, fileabspath])
+    except Exception as e:
+        logger.error(f"Failed to open audio in external program: {e}")
+        return False
+
+def is_connected():
+    try:
+        # Attempt to connect to a well-known host
+        socket.create_connection(("www.google.com", 80), timeout=2)
+        return True
+    except OSError:
+        return False
+
+
+TEXT_REPLACEMENTS_FILE = os.path.join(get_app_directory(), 'config', 'text_replacements.json')
+OCR_REPLACEMENTS_FILE = os.path.join(get_app_directory(), 'config', 'ocr_replacements.json')
+os.makedirs(os.path.dirname(TEXT_REPLACEMENTS_FILE), exist_ok=True)
+
+# if not os.path.exists(OCR_REPLACEMENTS_FILE):
+#     url = "https://raw.githubusercontent.com/bpwhelan/GameSentenceMiner/refs/heads/main/electron-src/assets/ocr_replacements.json"
+#     try:
+#         with urllib.request.urlopen(url) as response:
+#             data = response.read().decode('utf-8')
+#             with open(OCR_REPLACEMENTS_FILE, 'w', encoding='utf-8') as f:
+#                 f.write(data)
+#     except Exception as e:
+#         logger.error(f"Failed to fetch JSON from {url}: {e}")
+
+
+# Remove GitHub replacements from local OCR replacements file, these replacements are not needed
+def remove_github_replacements_from_local_ocr():
+    github_url = "https://raw.githubusercontent.com/bpwhelan/GameSentenceMiner/main/electron-src/assets/ocr_replacements.json"
+
+    github_replacements = {}
+    try:
+        response = requests.get(github_url)
+        response.raise_for_status()
+        github_data = response.json()
+        github_replacements = github_data.get('args', {}).get('replacements', {})
+        logger.debug(f"Successfully fetched {len(github_replacements)} replacements from GitHub.")
+    except requests.exceptions.RequestException as e:
+        logger.debug(f"Failed to fetch GitHub replacements from {github_url}: {e}")
+        return
+    except json.JSONDecodeError as e:
+        logger.debug(f"Error decoding JSON from GitHub response: {e}")
+        return
+
+    if not os.path.exists(OCR_REPLACEMENTS_FILE):
+        logger.debug(f"Local file {OCR_REPLACEMENTS_FILE} does not exist. No replacements to remove.")
+        return
+
+    try:
+        with open(OCR_REPLACEMENTS_FILE, 'r', encoding='utf-8') as f:
+            local_ocr_data = json.load(f)
+
+        local_replacements = local_ocr_data.get('args', {}).get('replacements', {})
+        original_count = len(local_replacements)
+        logger.debug(f"Loaded {original_count} replacements from local file.")
+
+        removed_count = 0
+        for key_to_remove in github_replacements.keys():
+            if key_to_remove in local_replacements:
+                del local_replacements[key_to_remove]
+                removed_count += 1
+
+        if removed_count > 0:
+            local_ocr_data['args']['replacements'] = local_replacements
+            with open(OCR_REPLACEMENTS_FILE, 'w', encoding='utf-8') as f:
+                json.dump(local_ocr_data, f, ensure_ascii=False, indent=4)
+            logger.debug(f"Successfully removed {removed_count} replacements from {OCR_REPLACEMENTS_FILE}.")
+            logger.debug(f"Remaining replacements in local file: {len(local_replacements)}")
+        else:
+            logger.debug("No matching replacements from GitHub found in your local file to remove.")
+
+    except json.JSONDecodeError as e:
+        logger.debug(f"Error decoding JSON from {OCR_REPLACEMENTS_FILE}: {e}. Please ensure it's valid JSON.")
+    except Exception as e:
+        logger.debug(f"An unexpected error occurred while processing {OCR_REPLACEMENTS_FILE}: {e}")
+
+
+remove_github_replacements_from_local_ocr()