GameSentenceMiner 2.19.5__py3-none-any.whl → 2.19.7__py3-none-any.whl

GameSentenceMiner/ui/screenshot_selector.py

@@ -1,4 +1,6 @@
+import math
 import os
+import re
 import subprocess
 import json
 import tkinter as tk
@@ -6,8 +8,9 @@ from tkinter import messagebox
 import ttkbootstrap as ttk
 from PIL import Image, ImageTk
 
+from GameSentenceMiner.util import ffmpeg
 from GameSentenceMiner.util.gsm_utils import sanitize_filename
-from GameSentenceMiner.util.configuration import get_temporary_directory, logger, ffmpeg_base_command_list, get_ffprobe_path
+from GameSentenceMiner.util.configuration import get_config, get_temporary_directory, logger, ffmpeg_base_command_list, get_ffprobe_path, ffmpeg_base_command_list_info
 
 
 class ScreenshotSelectorDialog(tk.Toplevel):
@@ -65,7 +68,7 @@ class ScreenshotSelectorDialog(tk.Toplevel):
         # Force always on top to ensure visibility
 
     def _extract_frames(self, video_path, timestamp, mode):
-        """Extracts frames using ffmpeg. Encapsulated from the original script."""
+        """Extracts frames using ffmpeg, with automatic black bar removal."""
         temp_dir = os.path.join(
             get_temporary_directory(False),
             "screenshot_frames",
@@ -87,17 +90,36 @@ class ScreenshotSelectorDialog(tk.Toplevel):
             logger.warning(f"Timestamp {timestamp_number} exceeds video duration {video_duration}.")
             return [], None
 
+        video_filters = []
+
+        if get_config().screenshot.trim_black_bars_wip:
+            crop_filter = ffmpeg.find_black_bars(video_path, timestamp_number)
+            if crop_filter:
+                video_filters.append(crop_filter)
+
+        # Always add the frame extraction filter
+        video_filters.append(f"fps=1/{0.25}")
+
         try:
+            # Build the final command for frame extraction
             command = ffmpeg_base_command_list + [
-                "-y",
+                "-y",                          # Overwrite output files without asking
                 "-ss", str(timestamp_number),
-                "-i", video_path,
-                "-vf", f"fps=1/{0.25}",
+                "-i", video_path
+            ]
+            
+            # Chain all collected filters (crop and fps) together with a comma
+            command.extend(["-vf", ",".join(video_filters)])
+            
+            command.extend([
                 "-vframes", "20",
                 os.path.join(temp_dir, "frame_%02d.png")
-            ]
+            ])
+            
+            logger.debug(f"Executing frame extraction command: {' '.join(command)}")
             subprocess.run(command, check=True, capture_output=True, text=True)
 
+            # The rest of your logic remains the same
             for i in range(1, 21):
                 frame_path = os.path.join(temp_dir, f"frame_{i:02d}.png")
                 if os.path.exists(frame_path):
@@ -122,7 +144,7 @@ class ScreenshotSelectorDialog(tk.Toplevel):
         except Exception as e:
             logger.error(f"An unexpected error occurred during frame extraction: {e}")
             return [], None
-
+        
     def _build_image_grid(self, image_paths, golden_frame):
         """Creates and displays the grid of selectable images."""
         self.images = [] # Keep a reference to images to prevent garbage collection

GameSentenceMiner/util/configuration.py

@@ -12,7 +12,7 @@ from logging.handlers import RotatingFileHandler
 from os.path import expanduser
 from sys import platform
 import time
-from typing import List, Dict
+from typing import Any, List, Dict
 import sys
 from enum import Enum
 
@@ -59,6 +59,28 @@ supported_formats = {
     'm4a': 'aac',
 }
 
+KNOWN_ASPECT_RATIOS = [
+    # --- Classic / Legacy ---
+    {"name": "4:3 (SD / Retro Games)", "ratio": 4 / 3},
+    {"name": "5:4 (Old PC Monitors)", "ratio": 5 / 4},
+    {"name": "3:2 (Handheld / GBA / DS / DSLR)", "ratio": 3 / 2},
+
+    # --- Modern Displays ---
+    {"name": "16:10 (PC Widescreen)", "ratio": 16 / 10},
+    {"name": "16:9 (Standard HD / 1080p / 4K)", "ratio": 16 / 9},
+    {"name": "18:9 (Mobile / Some Modern Laptops)", "ratio": 18 / 9},
+    {"name": "19.5:9 (Modern Smartphones)", "ratio": 19.5 / 9},
+    {"name": "21:9 (UltraWide)", "ratio": 21 / 9},
+    {"name": "24:10 (UltraWide+)", "ratio": 24 / 10},
+    {"name": "32:9 (Super UltraWide)", "ratio": 32 / 9},
+
+    # --- Vertical / Mobile ---
+    {"name": "9:16 (Portrait Mode)", "ratio": 9 / 16},
+    {"name": "3:4 (Portrait 4:3)", "ratio": 3 / 4},
+    {"name": "1:1 (Square / UI Capture)", "ratio": 1 / 1},
+]
+
+KNOWN_ASPECT_RATIOS_DICT = {item["name"]: item["ratio"] for item in KNOWN_ASPECT_RATIOS}
 
 def is_linux():
     return platform == 'linux'
@@ -490,6 +512,7 @@ class Screenshot:
     use_new_screenshot_logic: bool = False
     screenshot_timing_setting: str = 'beginning'  # 'middle', 'end'
     use_screenshot_selector: bool = False
+    trim_black_bars_wip: bool = True
 
     def __post_init__(self):
         if not self.screenshot_timing_setting and self.use_beginning_of_line_as_screenshot:
@@ -632,6 +655,7 @@ class Ai:
     use_canned_translation_prompt: bool = True
     use_canned_context_prompt: bool = False
     custom_prompt: str = ''
+    custom_texthooker_prompt: str = ''
     dialogue_context_length: int = 10
 
     def __post_init__(self):
@@ -1321,10 +1345,11 @@ class AnkiUpdateResult:
     video_in_anki: str = ''
     word_path: str = ''
     word: str = ''
+    extra_tags: List[str] = field(default_factory=list)
 
     @staticmethod
     def failure():
-        return AnkiUpdateResult(success=False, audio_in_anki='', screenshot_in_anki='', prev_screenshot_in_anki='', sentence_in_anki='', multi_line=False, video_in_anki='', word_path='', word='')
+        return AnkiUpdateResult(success=False, audio_in_anki='', screenshot_in_anki='', prev_screenshot_in_anki='', sentence_in_anki='', multi_line=False, video_in_anki='', word_path='', word='', extra_tags=[])
 
 
 @dataclass_json
@@ -1376,6 +1401,8 @@ def get_ffprobe_path():
 
 ffmpeg_base_command_list = [get_ffmpeg_path(), "-hide_banner", "-loglevel", "error", '-nostdin']
 
+ffmpeg_base_command_list_info = [get_ffmpeg_path(), "-hide_banner", "-loglevel", "info", '-nostdin']
+
 
 # logger.debug(f"Running in development mode: {is_dev}")
 # logger.debug(f"Running on Beangate's PC: {is_beangate}")

GameSentenceMiner/util/ffmpeg.py

@@ -1,5 +1,6 @@
 import json
 import os
+import re
 import subprocess
 import sys
 import tempfile
@@ -12,8 +13,8 @@ import shutil
 
 from GameSentenceMiner import obs
 from GameSentenceMiner.ui.config_gui import ConfigApp
-from GameSentenceMiner.util.configuration import ffmpeg_base_command_list, get_ffprobe_path, logger, get_config, \
-    get_temporary_directory, gsm_state, is_linux
+from GameSentenceMiner.util.configuration import ffmpeg_base_command_list, get_ffprobe_path, get_master_config, logger, get_config, \
+    get_temporary_directory, gsm_state, is_linux, ffmpeg_base_command_list_info, KNOWN_ASPECT_RATIOS
 from GameSentenceMiner.util.gsm_utils import make_unique_file_name, get_file_modification_time
 from GameSentenceMiner.util import configuration
 from GameSentenceMiner.util.text_log import initial_time
@@ -223,53 +224,307 @@ def get_screenshot(video_file, screenshot_timing, try_selector=False):
             return output
         else:
             logger.error("Frame extractor script failed to run or returned no output, defaulting")
+
     output_image = make_unique_file_name(os.path.join(
         get_temporary_directory(), f"{obs.get_current_game(sanitize=True)}.{get_config().screenshot.extension}"))
-    # FFmpeg command to extract the last frame of the video
+    
+    # Base command for extracting the frame
     ffmpeg_command = ffmpeg_base_command_list + [
         "-ss", f"{screenshot_timing}",
         "-i", f"{video_file}",
         "-vframes", "1"  # Extract only one frame
     ]
+    
+    video_filters = []
+
+    if get_config().screenshot.trim_black_bars_wip:
+        crop_filter = find_black_bars(video_file, screenshot_timing)
+        if crop_filter:
+            video_filters.append(crop_filter)
+
+    if get_config().screenshot.width or get_config().screenshot.height:
+        # Add scaling to the filter chain
+        scale_filter = f"scale={get_config().screenshot.width or -1}:{get_config().screenshot.height or -1}"
+        video_filters.append(scale_filter)
+
+    # If we have any filters (crop, scale, etc.), chain them together with commas
+    if video_filters:
+        ffmpeg_command.extend(["-vf", ",".join(video_filters)])
 
     if get_config().screenshot.custom_ffmpeg_settings:
         ffmpeg_command.extend(get_config().screenshot.custom_ffmpeg_settings.replace("\"", "").split(" "))
     else:
-        ffmpeg_command.extend(["-compression_level", "6", "-q:v", get_config().screenshot.quality])
-
-    if get_config().screenshot.width or get_config().screenshot.height:
-        ffmpeg_command.extend(
-            ["-vf", f"scale={get_config().screenshot.width or -1}:{get_config().screenshot.height or -1}"])
+        # Ensure quality settings are strings
+        ffmpeg_command.extend(["-compression_level", "6", "-q:v", str(get_config().screenshot.quality)])
 
     ffmpeg_command.append(f"{output_image}")
 
-    logger.debug(f"FFMPEG SS Command: {ffmpeg_command}")
+    logger.debug(f"FFMPEG SS Command: {' '.join(map(str, ffmpeg_command))}")
 
     try:
+        # Changed the retry loop to be more robust
         for i in range(3):
-            logger.debug(" ".join(ffmpeg_command))
-            result = subprocess.run(ffmpeg_command)
-            if result.returncode != 0 and i < 2:
-                raise RuntimeError(f"FFmpeg command failed with return code {result.returncode}")
-            else:
-                break
+            logger.debug("Executing FFmpeg command...")
+            result = subprocess.run(ffmpeg_command, capture_output=True, text=True)
+            if result.returncode == 0:
+                break # Success!
+            logger.warning(f"FFmpeg attempt {i+1} failed. Stderr: {result.stderr}")
+            if i == 2: # Last attempt failed
+                 raise RuntimeError(f"FFmpeg command failed after 3 attempts. Stderr: {result.stderr}")
     except Exception as e:
         logger.error(f"Error running FFmpeg command: {e}. Defaulting to standard PNG.")
         output_image = make_unique_file_name(os.path.join(
             get_temporary_directory(),
             f"{obs.get_current_game(sanitize=True)}.png"))
-        ffmpeg_command = ffmpeg_base_command_list + [
+        # Fallback command without any complex filters
+        fallback_command = ffmpeg_base_command_list + [
             "-ss", f"{screenshot_timing}",
             "-i", video_file,
             "-vframes", "1",
             output_image
         ]
-        subprocess.run(ffmpeg_command)
+        subprocess.run(fallback_command)
 
     logger.debug(f"Screenshot saved to: {output_image}")
 
     return output_image
 
+def get_video_dimensions(video_file):
+    """Get the width and height of a video file."""
+    try:
+        ffprobe_command = [
+            get_ffprobe_path(),
+            "-v", "error",
+            "-select_streams", "v:0",
+            "-show_entries", "stream=width,height",
+            "-of", "json",
+            video_file
+        ]
+        
+        result = subprocess.run(
+            ffprobe_command,
+            capture_output=True,
+            text=True,
+            check=True
+        )
+        
+        output = json.loads(result.stdout)
+        width = output['streams'][0]['width']
+        height = output['streams'][0]['height']
+        return width, height
+    except Exception as e:
+        logger.error(f"Error getting video dimensions: {e}")
+        return None, None
+
+# How close the detected ratio needs to be to a known ratio to snap (e.g., 0.05 = 5%)
+RATIO_TOLERANCE = 0.05
+
+def _calculate_target_crop(orig_width, orig_height, target_ratio):
+    """
+    Calculates the new dimensions and offsets for a target aspect ratio.
+    
+    Returns: A tuple (new_width, new_height, x_offset, y_offset)
+    """
+    orig_ratio = orig_width / orig_height
+
+    if abs(orig_ratio - target_ratio) < 0.01: # Already at the target ratio
+        return orig_width, orig_height, 0, 0
+
+    if orig_ratio > target_ratio:
+        # Original is wider than target (pillarbox scenario)
+        # Keep original height, calculate new width
+        new_width = round(orig_height * target_ratio)
+        new_height = orig_height
+        x_offset = round((orig_width - new_width) / 2)
+        y_offset = 0
+    else:
+        # Original is narrower than target (letterbox scenario)
+        # Keep original width, calculate new height
+        new_width = orig_width
+        new_height = round(orig_width / target_ratio)
+        x_offset = 0
+        y_offset = round((orig_height - new_height) / 2)
+
+    # Ensure dimensions are even for compatibility
+    new_width = new_width if new_width % 2 == 0 else new_width - 1
+    new_height = new_height if new_height % 2 == 0 else new_height - 1
+    x_offset = x_offset if x_offset % 2 == 0 else x_offset - 1
+    y_offset = y_offset if y_offset % 2 == 0 else y_offset - 1
+    
+    return new_width, new_height, x_offset, y_offset
+
+
+def find_black_bars_with_ratio_snapping(video_file, screenshot_timing):
+    logger.info("Attempting to detect black bars with aspect ratio snapping...")
+    crop_filter = None
+    try:
+        orig_width, orig_height = get_video_dimensions(video_file)
+        if not orig_width or not orig_height:
+            logger.warning("Could not determine video dimensions. Skipping black bar detection.")
+            return None
+        
+        orig_aspect = orig_width / orig_height
+        logger.debug(f"Original video dimensions: {orig_width}x{orig_height} (Ratio: {orig_aspect:.3f})")
+        
+        cropdetect_command = ffmpeg_base_command_list_info + [
+            "-i", video_file,
+            "-ss", f"{screenshot_timing}",
+            "-t", "5", # Analyze for 5 seconds
+            "-vf", "cropdetect=limit=16", # limit=16 for near-black detection, 24 is too aggressive
+            "-f", "null", "-"
+        ]
+        
+        result = subprocess.run(
+            cropdetect_command, 
+            capture_output=True, 
+            text=True, 
+            check=False
+        )
+        
+        crop_lines = re.findall(r"crop=\d+:\d+:\d+:\d+", result.stderr)
+        if not crop_lines:
+            logger.info("cropdetect did not find any black bars to remove.")
+            return None
+            
+        last_crop_params = crop_lines[-1]
+        match = re.match(r"crop=(\d+):(\d+):(\d+):(\d+)", last_crop_params)
+        if not match:
+            logger.warning(f"Could not parse cropdetect output: {last_crop_params}")
+            return None
+
+        detected_width = int(match.group(1))
+        detected_height = int(match.group(2))
+        
+        if detected_width == orig_width and detected_height == orig_height:
+            logger.info("cropdetect suggests no cropping is needed.")
+            return None
+
+        detected_aspect = detected_width / detected_height
+        logger.debug(f"cropdetect suggests crop to: {detected_width}x{detected_height} (Ratio: {detected_aspect:.3f})")
+
+        best_match = None
+        min_diff = float('inf')
+
+        for known in KNOWN_ASPECT_RATIOS:
+            diff = abs(detected_aspect - known["ratio"]) / known["ratio"]
+            if diff < min_diff:
+                min_diff = diff
+                best_match = known
+        
+        get_master_config().scenes_info
+        
+        if best_match and min_diff <= RATIO_TOLERANCE:
+            target_name = best_match["name"]
+            target_ratio = best_match["ratio"]
+            logger.info(
+                f"Detected ratio ({detected_aspect:.3f}) is close to {target_name} ({target_ratio:.3f}). "
+                f"Snapping to the standard ratio."
+            )
+            
+            crop_width, crop_height, crop_x, crop_y = _calculate_target_crop(
+                orig_width, orig_height, target_ratio
+            )
+            
+            area_ratio = (crop_width * crop_height) / (orig_width * orig_height)
+            if area_ratio < 0.50:
+                logger.warning(
+                    f"Calculated crop would remove too much video ({1 - area_ratio:.1%}). "
+                    "Skipping crop to avoid false detection."
+                )
+                return None
+            
+            crop_filter = f"crop={crop_width}:{crop_height}:{crop_x}:{crop_y}"
+            logger.info(f"Applying snapped aspect ratio filter: {crop_filter}")
+            
+        else:
+            logger.info(
+                f"Detected crop ratio ({detected_aspect:.3f}) is not close enough to any known standard. "
+                "Skipping crop to avoid non-standard results."
+            )
+            return None
+
+    except Exception as e:
+        logger.error(f"Error during black bar detection: {e}. Proceeding without cropping.")
+    
+    return crop_filter
+
+def find_black_bars(video_file, screenshot_timing):
+    logger.info("Attempting to detect black bars...")
+    crop_filter = None
+    try:
+        # Get original video dimensions
+        orig_width, orig_height = get_video_dimensions(video_file)
+        if not orig_width or not orig_height:
+            logger.warning("Could not determine video dimensions. Skipping black bar detection.")
+            return None
+        
+        logger.debug(f"Original video dimensions: {orig_width}x{orig_height}")
+        
+        cropdetect_command = ffmpeg_base_command_list_info + [
+            "-i", video_file,
+            "-ss", f"{screenshot_timing}", # Start near the screenshot time
+            "-t", "1",                     # Analyze for 1 second
+            "-vf", "cropdetect=limit=16",  # limit=0 means true black only, round=2 for even dimensions
+            "-f", "null", "-"              # Discard video output
+        ]
+        
+        result = subprocess.run(
+            cropdetect_command, 
+            capture_output=True, 
+            text=True, 
+            check=False
+        )
+        
+        crop_lines = re.findall(r"crop=\d+:\d+:\d+:\d+", result.stderr)
+        if crop_lines:
+            crop_params = crop_lines[-1]
+            print(crop_params)
+            # Parse crop parameters: crop=width:height:x:y
+            match = re.match(r"crop=(\d+):(\d+):(\d+):(\d+)", crop_params)
+            if match:
+                crop_width = int(match.group(1))
+                crop_height = int(match.group(2))
+                
+                # Calculate what percentage of the original video would remain
+                area_ratio = (crop_width * crop_height) / (orig_width * orig_height)
+                
+                # Calculate aspect ratios
+                orig_aspect = orig_width / orig_height
+                crop_aspect = crop_width / crop_height
+                aspect_diff = abs(orig_aspect - crop_aspect) / orig_aspect
+                
+                logger.debug(f"Crop would be {crop_width}x{crop_height} ({area_ratio:.1%} of original area)")
+                logger.debug(f"Original aspect ratio: {orig_aspect:.3f}, Crop aspect ratio: {crop_aspect:.3f}, Difference: {aspect_diff:.1%}")
+                
+                # Safeguards:
+                # 1. Crop must retain at least 25% of the original video area
+                # 2. Aspect ratio must not change by more than 30%
+                if area_ratio < 0.25:
+                    logger.warning(f"Crop would remove too much of the video ({area_ratio:.1%} remaining). Skipping crop to avoid false detection.")
+                    return None
+                
+                if aspect_diff > 0.30:
+                    for ratio in KNOWN_ASPECT_RATIOS:
+                        known_ratio = ratio["ratio"]
+                        known_diff = abs(crop_aspect - known_ratio) / known_ratio
+                        if known_diff < RATIO_TOLERANCE:
+                            logger.info(f"Crop aspect ratio ({crop_aspect:.3f}) is close to known ratio {ratio['name']} ({known_ratio:.3f}). Accepting crop.")
+                            break
+                    else:
+                        logger.warning(f"Crop would significantly change aspect ratio ({aspect_diff:.1%} difference). Skipping crop to avoid false detection.")
+                        return None
+                
+                crop_filter = crop_params
+                logger.info(f"Detected valid black bars. Applying filter: {crop_filter}")
+            else:
+                logger.warning("Could not parse crop parameters.")
+        else:
+            logger.debug("cropdetect did not find any black bars to remove.")
+            
+    except Exception as e:
+        logger.error(f"Error during black bar detection: {e}. Proceeding without cropping.")
+    return crop_filter
+
 def get_screenshot_for_line(video_file, game_line, try_selector=False):
     return get_screenshot(video_file, get_screenshot_time(video_file, game_line), try_selector)
 

GameSentenceMiner/web/gsm_websocket.py

@@ -77,10 +77,12 @@ class WebsocketServerThread(threading.Thread):
             self._event.set()
             while True:
                 try:
-                    self.server = start_server = websockets.serve(self.server_handler,
-                                                                  get_config().advanced.localhost_bind_address,
-                                                                  self.get_ws_port_func(),
-                                                                  max_size=1000000000)
+                    self.server = start_server = websockets.serve(
+                        self.server_handler,
+                        get_config().advanced.localhost_bind_address,
+                        self.get_ws_port_func(),
+                        max_size=1000000000,
+                    )
                     async with start_server:
                         await stop_event.wait()
                     return
@@ -106,11 +108,11 @@ websocket_server_thread.start()
 plaintext_websocket_server_thread = None
 if get_config().advanced.plaintext_websocket_port:
     plaintext_websocket_server_thread = WebsocketServerThread(
-        read=False, get_ws_port_func=lambda: get_config().get_field_value('advanced', 'plaintext_websocket_port'))
+        read=True, get_ws_port_func=lambda: get_config().get_field_value('advanced', 'plaintext_websocket_port'))
     plaintext_websocket_server_thread.start()
 
 overlay_server_thread = WebsocketServerThread(
-    read=False, get_ws_port_func=lambda: get_config().get_field_value('overlay', 'websocket_port'))
+    read=True, get_ws_port_func=lambda: get_config().get_field_value('overlay', 'websocket_port'))
 overlay_server_thread.start()
 
 websocket_server_threads = [