GameSentenceMiner 2.16.8__py3-none-any.whl → 2.16.9__py3-none-any.whl

GameSentenceMiner/config_gui.py

@@ -363,6 +363,8 @@ class ConfigApp:
         self.vad_trim_beginning_value = tk.BooleanVar(value=self.settings.vad.trim_beginning)
         self.vad_beginning_offset_value = tk.StringVar(value=str(self.settings.vad.beginning_offset))
         self.add_audio_on_no_results_value = tk.BooleanVar(value=self.settings.vad.add_audio_on_no_results)
+        self.use_tts_as_fallback_value = tk.BooleanVar(value=self.settings.vad.use_tts_as_fallback)
+        self.tts_url_value = tk.StringVar(value=self.settings.vad.tts_url)
         self.language_value = tk.StringVar(value=self.settings.vad.language)
         self.cut_and_splice_segments_value = tk.BooleanVar(value=self.settings.vad.cut_and_splice_segments)
         self.splice_padding_value = tk.StringVar(value=str(self.settings.vad.splice_padding) if self.settings.vad.splice_padding else "")
@@ -397,6 +399,8 @@ class ConfigApp:
         self.overlay_websocket_port_value = tk.StringVar(value=str(self.settings.overlay.websocket_port))
         self.overlay_websocket_send_value = tk.BooleanVar(value=self.settings.overlay.monitor_to_capture)
         self.overlay_engine_value = tk.StringVar(value=self.settings.overlay.engine)
+        self.periodic_value = tk.BooleanVar(value=self.settings.overlay.periodic)
+        self.periodic_interval_value = tk.StringVar(value=str(self.settings.overlay.periodic_interval))
         
         # Master Config Settings
         self.switch_to_default_if_not_found_value = tk.BooleanVar(value=self.master_config.switch_to_default_if_not_found)
@@ -595,6 +599,8 @@ class ConfigApp:
                 trim_beginning=self.vad_trim_beginning_value.get(),
                 beginning_offset=float(self.vad_beginning_offset_value.get()),
                 add_audio_on_no_results=self.add_audio_on_no_results_value.get(),
+                use_tts_as_fallback=self.use_tts_as_fallback_value.get(),
+                tts_url=self.tts_url_value.get(),
                 language=self.language_value.get(),
                 cut_and_splice_segments=self.cut_and_splice_segments_value.get(),
                 splice_padding=float(self.splice_padding_value.get()) if self.splice_padding_value.get() else 0.0,
@@ -630,7 +636,9 @@ class ConfigApp:
             overlay=Overlay(
                 websocket_port=int(self.overlay_websocket_port_value.get()),
                 monitor_to_capture=self.overlay_monitor.current() if self.monitors else 0,
-                engine=OverlayEngine(self.overlay_engine_value.get()).value if self.overlay_engine_value.get() else OverlayEngine.LENS.value
+                engine=OverlayEngine(self.overlay_engine_value.get()).value if self.overlay_engine_value.get() else OverlayEngine.LENS.value,
+                periodic=self.periodic_value.get(),
+                periodic_interval=self.periodic_interval_value.get(),
             )
             # wip=WIP(
             #     overlay_websocket_port=int(self.overlay_websocket_port_value.get()),
@@ -1111,6 +1119,17 @@ class ConfigApp:
             row=self.current_row, column=1, sticky='W', pady=2)
         self.current_row += 1
 
+        # TODO ADD LOCALIZATION
+        tts_fallback_i18n = vad_i18n.get('use_tts_as_fallback', {})
+        HoverInfoLabelWidget(vad_frame, text=tts_fallback_i18n.get('label', 'Use TTS as Fallback.'), tooltip=tts_fallback_i18n.get('tooltip', 'Use TTS if no audio is detected'), row=self.current_row, column=0)
+        ttk.Checkbutton(vad_frame, variable=self.use_tts_as_fallback_value, bootstyle="round-toggle").grid(row=self.current_row, column=1, sticky='W', pady=2)
+        self.current_row += 1
+
+        tts_url_i18n = vad_i18n.get('tts_url', {})
+        HoverInfoLabelWidget(vad_frame, text=tts_url_i18n.get('label', 'TTS URL'), tooltip=tts_url_i18n.get('tooltip', 'The URL for the TTS service'), row=self.current_row, column=0)
+        ttk.Entry(vad_frame, textvariable=self.tts_url_value).grid(row=self.current_row, column=1, sticky='EW', pady=2)
+        self.current_row += 1
+
         end_offset_i18n = vad_i18n.get('audio_end_offset', {})
         HoverInfoLabelWidget(vad_frame, text=end_offset_i18n.get('label', '...'),
                              tooltip=end_offset_i18n.get('tooltip', '...'), foreground="dark orange",
@@ -1161,6 +1180,13 @@ class ConfigApp:
         # Add Reset Button
         self.add_reset_button(vad_frame, "vad", self.current_row, column=0, recreate_tab=self.create_vad_tab)
 
+        for col in range(3):
+            vad_frame.grid_columnconfigure(col, weight=0)
+        for row in range(self.current_row):
+            vad_frame.grid_rowconfigure(row, minsize=30)
+
+        return vad_frame
+
     @new_tab
     def create_paths_tab(self):
         if self.paths_tab is None:
@@ -1183,7 +1209,7 @@ class ConfigApp:
         ttk.Button(paths_frame, text=browse_text, command=lambda: self.browse_folder(folder_watch_entry),
                    bootstyle="outline").grid(row=self.current_row, column=2, padx=5, pady=2)
         self.current_row += 1
-        
+
         # Combine "Copy temp files to output folder" and "Output folder" on one row
         copy_to_output_i18n = paths_i18n.get('copy_temp_files_to_output_folder', {})
         combined_i18n = paths_i18n.get('output_folder', {})
@@ -2058,7 +2084,7 @@ class ConfigApp:
         entry = ttk.Entry(ai_frame, textvariable=self.open_ai_url_value)
         entry.grid(row=self.current_row, column=1, sticky='EW', pady=2)
         self.current_row += 1
-        
+
         entry.bind("<FocusOut>", lambda e, row=self.current_row: self.update_models_element(ai_frame, row))
         entry.bind("<Return>", lambda e, row=self.current_row: self.update_models_element(ai_frame, row))
 
@@ -2253,6 +2279,21 @@ class ConfigApp:
                                            textvariable=self.overlay_engine_value)
         self.overlay_engine.grid(row=self.current_row, column=1, sticky='EW', pady=2)
         self.current_row += 1
+        
+        # Periodic Settings
+        periodic_i18n = overlay_i18n.get('periodic', {})
+        HoverInfoLabelWidget(overlay_frame, text=periodic_i18n.get('label', 'Periodic:'),
+                             tooltip=periodic_i18n.get('tooltip', 'Enable periodic Scanning.'),
+                             row=self.current_row, column=0)
+        ttk.Checkbutton(overlay_frame, variable=self.periodic_value, bootstyle="round-toggle").grid(
+            row=self.current_row, column=1, sticky='W', pady=2)
+        self.current_row += 1
+        periodic_interval_i18n = overlay_i18n.get('periodic_interval', {})
+        HoverInfoLabelWidget(overlay_frame, text=periodic_interval_i18n.get('label', 'Periodic Interval:'),
+                             tooltip=periodic_interval_i18n.get('tooltip', 'Interval for periodic scanning.'),
+                             row=self.current_row, column=0)
+        ttk.Entry(overlay_frame, textvariable=self.periodic_interval_value).grid(row=self.current_row, column=1, sticky='EW', pady=2)
+        self.current_row += 1
 
         if self.monitors:
             # Ensure the index is valid
@@ -2293,7 +2334,7 @@ class ConfigApp:
             # self.controller_hotkey_entry.grid(row=self.current_row, column=1, sticky='EW', pady=2)
             
             # listen_for_input_button = ttk.Button(wip_frame, text="Listen for Input", command=lambda: self.listen_for_controller_input())
-            # listen_for_input_button.grid(row=self.current_row, column=2, sticky='EW', pady=2)
+            # listen_for_input_button.grid(row=self.current_row, column=2, sticky='EW', pady=2, padx=5)
             # self.current_row += 1
 
         except Exception as e:
@@ -2406,6 +2447,7 @@ class ConfigApp:
             default_path = get_default_anki_media_collection_path()
             if default_path != self.anki_media_collection_value.get():
                 self.anki_media_collection_value.set(default_path)
+               
                 self.save_settings()
 
 

GameSentenceMiner/gsm.py

@@ -1,3 +1,4 @@
+import tempfile
 import time
 import asyncio
 import subprocess
@@ -6,6 +7,11 @@ import sys
 import os
 import warnings
 
+import requests
+
+from GameSentenceMiner.util.get_overlay_coords import OverlayThread
+from GameSentenceMiner.util.gsm_utils import remove_html_and_cloze_tags
+
 os.environ.pop('TCL_LIBRARY', None)
 
 
@@ -21,6 +27,7 @@ def handle_error_in_initialization(e):
         logger.info("Exiting due to initialization error.")
         sys.exit(1)
 
+
 try:
     import os.path
     import signal
@@ -48,15 +55,18 @@ try:
 
     start_time = time.time()
     from GameSentenceMiner.util.downloader.download_tools import download_obs_if_needed, download_ffmpeg_if_needed
-    logger.debug(f"[Import] download_tools (download_obs_if_needed, download_ffmpeg_if_needed): {time.time() - start_time:.3f}s")
+    logger.debug(
+        f"[Import] download_tools (download_obs_if_needed, download_ffmpeg_if_needed): {time.time() - start_time:.3f}s")
 
     start_time = time.time()
     from GameSentenceMiner.util.communication.send import send_restart_signal
-    logger.debug(f"[Import] send_restart_signal: {time.time() - start_time:.3f}s")
+    logger.debug(
+        f"[Import] send_restart_signal: {time.time() - start_time:.3f}s")
 
     start_time = time.time()
     from GameSentenceMiner.util.gsm_utils import wait_for_stable_file, make_unique_file_name, run_new_thread
-    logger.debug(f"[Import] gsm_utils (wait_for_stable_file, make_unique_file_name, run_new_thread): {time.time() - start_time:.3f}s")
+    logger.debug(
+        f"[Import] gsm_utils (wait_for_stable_file, make_unique_file_name, run_new_thread): {time.time() - start_time:.3f}s")
 
     start_time = time.time()
     from GameSentenceMiner import anki
@@ -68,7 +78,8 @@ try:
 
     start_time = time.time()
     from GameSentenceMiner.util import configuration, notification, ffmpeg
-    logger.debug(f"[Import] util (configuration, notification, ffmpeg): {time.time() - start_time:.3f}s")
+    logger.debug(
+        f"[Import] util (configuration, notification, ffmpeg): {time.time() - start_time:.3f}s")
 
     start_time = time.time()
     from GameSentenceMiner import gametext
@@ -84,19 +95,23 @@ try:
 
     start_time = time.time()
     from GameSentenceMiner.util.communication.websocket import connect_websocket, register_websocket_message_handler, FunctionName
-    logger.debug(f"[Import] websocket (connect_websocket, register_websocket_message_handler, FunctionName): {time.time() - start_time:.3f}s")
+    logger.debug(
+        f"[Import] websocket (connect_websocket, register_websocket_message_handler, FunctionName): {time.time() - start_time:.3f}s")
 
     start_time = time.time()
     from GameSentenceMiner.util.ffmpeg import get_audio_and_trim, get_video_timings, get_ffmpeg_path
-    logger.debug(f"[Import] util.ffmpeg (get_audio_and_trim, get_video_timings, get_ffmpeg_path): {time.time() - start_time:.3f}s")
+    logger.debug(
+        f"[Import] util.ffmpeg (get_audio_and_trim, get_video_timings, get_ffmpeg_path): {time.time() - start_time:.3f}s")
 
     start_time = time.time()
     from GameSentenceMiner.obs import check_obs_folder_is_correct
-    logger.debug(f"[Import] obs.check_obs_folder_is_correct: {time.time() - start_time:.3f}s")
+    logger.debug(
+        f"[Import] obs.check_obs_folder_is_correct: {time.time() - start_time:.3f}s")
 
     start_time = time.time()
     from GameSentenceMiner.util.text_log import GameLine, get_text_event, get_mined_line, get_all_lines, game_log
-    logger.debug(f"[Import] util.text_log (GameLine, get_text_event, get_mined_line, get_all_lines, game_log): {time.time() - start_time:.3f}s")
+    logger.debug(
+        f"[Import] util.text_log (GameLine, get_text_event, get_mined_line, get_all_lines, game_log): {time.time() - start_time:.3f}s")
 
     start_time = time.time()
     from GameSentenceMiner.util import *
@@ -104,15 +119,18 @@ try:
 
     start_time = time.time()
     from GameSentenceMiner.web import texthooking_page
-    logger.debug(f"[Import] web.texthooking_page: {time.time() - start_time:.3f}s")
+    logger.debug(
+        f"[Import] web.texthooking_page: {time.time() - start_time:.3f}s")
 
     start_time = time.time()
     from GameSentenceMiner.web.service import handle_texthooker_button, set_get_audio_from_video_callback
-    logger.debug(f"[Import] web.service (handle_texthooker_button, set_get_audio_from_video_callback): {time.time() - start_time:.3f}s")
+    logger.debug(
+        f"[Import] web.service (handle_texthooker_button, set_get_audio_from_video_callback): {time.time() - start_time:.3f}s")
 
     start_time = time.time()
     from GameSentenceMiner.web.texthooking_page import run_text_hooker_page
-    logger.debug(f"[Import] web.texthooking_page.run_text_hooker_page: {time.time() - start_time:.3f}s")
+    logger.debug(
+        f"[Import] web.texthooking_page.run_text_hooker_page: {time.time() - start_time:.3f}s")
 except Exception as e:
     from GameSentenceMiner.util.configuration import logger, is_linux, is_windows
     handle_error_in_initialization(e)
@@ -172,8 +190,9 @@ class VideoToAudioHandler(FileSystemEventHandler):
                 if get_config().features.backfill_audio:
                     last_note = anki.get_cards_by_sentence(
                         gametext.current_line_after_regex)
-                    
-            note, last_note = anki.get_initial_card_info(last_note, selected_lines)
+
+            note, last_note = anki.get_initial_card_info(
+                last_note, selected_lines)
             tango = last_note.get_field(
                 get_config().anki.word_field) if last_note else ''
 
@@ -184,12 +203,15 @@ class VideoToAudioHandler(FileSystemEventHandler):
                 start_line = selected_lines[0]
                 mined_line = get_mined_line(last_note, selected_lines)
                 line_cutoff = selected_lines[-1].get_next_time()
+                full_text = remove_html_and_cloze_tags(note['fields'][get_config().anki.sentence_field])
             else:
                 mined_line = get_text_event(last_note)
                 if mined_line:
                     start_line = mined_line
                     if mined_line.next:
                         line_cutoff = mined_line.next.time
+                    full_text = mined_line.text
+
             gsm_state.last_mined_line = mined_line
 
             if os.path.exists(video_path) and os.access(video_path, os.R_OK):
@@ -213,7 +235,8 @@ class VideoToAudioHandler(FileSystemEventHandler):
                     line_cutoff,
                     video_path,
                     anki_card_creation_time,
-                    mined_line=mined_line)
+                    mined_line=mined_line,
+                    full_text=full_text)
             else:
                 final_audio_output = ""
                 vad_result = VADResult(True, 0, 0, '')
@@ -269,11 +292,13 @@ class VideoToAudioHandler(FileSystemEventHandler):
                     f"Error removing video file {video_path}: {e}", exc_info=True)
 
     @staticmethod
-    def get_audio(game_line, next_line_time, video_path, anki_card_creation_time=None, temporary=False, timing_only=False, mined_line=None):
+    def get_audio(game_line, next_line_time, video_path, anki_card_creation_time=None, temporary=False, timing_only=False, mined_line=None, full_text=''):
         trimmed_audio, start_time, end_time = get_audio_and_trim(
             video_path, game_line, next_line_time, anki_card_creation_time)
         if temporary:
             return ffmpeg.convert_audio_to_wav_lossless(trimmed_audio)
+        if not get_config().vad.do_vad_postprocessing:
+            return trimmed_audio, VADResult(True, start_time, end_time, "No VAD"), trimmed_audio, start_time, end_time
         vad_trimmed_audio = make_unique_file_name(
             f"{os.path.abspath(configuration.get_temporary_directory())}/{obs.get_current_game(sanitize=True)}.{get_config().audio.extension}")
         final_audio_output = make_unique_file_name(os.path.join(get_temporary_directory(),
@@ -283,6 +308,25 @@ class VideoToAudioHandler(FileSystemEventHandler):
             trimmed_audio, vad_trimmed_audio, game_line)
         if timing_only:
             return vad_result
+
+        if not vad_result.success:
+            if get_config().vad.add_audio_on_no_results:
+                logger.info("No voice activity detected, using full audio.")
+                vad_result.output_audio = trimmed_audio
+            elif get_config().vad.use_tts_as_fallback:
+                logger.info(
+                    "No voice activity detected, using TTS as fallback.")
+                text_to_tts = full_text if full_text else game_line.text
+                url = get_config().vad.tts_url.replace("$s", text_to_tts)
+                tts_resp = requests.get(url)
+                if not tts_resp.ok:
+                    logger.error(
+                        f"Error fetching TTS audio from {url}. Is it running?: {tts_resp.status_code} {tts_resp.text}")
+                with tempfile.NamedTemporaryFile(dir=get_temporary_directory(), delete=False, suffix=".opus") as tmpfile:
+                    tmpfile.write(tts_resp.content)
+                    vad_result.output_audio = tmpfile.name
+        else:
+            logger.info(vad_result.trim_successful_string())
         if vad_result.output_audio:
             vad_trimmed_audio = vad_result.output_audio
         if get_config().audio.ffmpeg_reencode_options_to_use and os.path.exists(vad_trimmed_audio):
@@ -404,12 +448,13 @@ def open_multimine(icon, item):
 
 
 def exit_program(passed_icon, item):
-        """Exit the application."""
-        if not passed_icon:
-            passed_icon = icon
-        logger.info("Exiting...")
-        passed_icon.stop()
-        cleanup()
+    """Exit the application."""
+    if not passed_icon:
+        passed_icon = icon
+    logger.info("Exiting...")
+    passed_icon.stop()
+    cleanup()
+
 
 class GSMTray(threading.Thread):
     def __init__(self):
@@ -421,12 +466,11 @@ class GSMTray(threading.Thread):
     def run(self):
         self.run_tray()
 
-
     def run_tray(self):
         self.profile_menu = Menu(
             *[MenuItem(("Active: " if profile == get_master_config().current_profile else "") + profile, self.switch_profile) for
-            profile in
-            get_master_config().get_all_profile_names()]
+              profile in
+              get_master_config().get_all_profile_names()]
         )
 
         menu = Menu(
@@ -447,8 +491,8 @@ class GSMTray(threading.Thread):
         # Recreate the menu with the updated button text
         profile_menu = Menu(
             *[MenuItem(("Active: " if profile == get_master_config().current_profile else "") + profile, self.switch_profile) for
-            profile in
-            get_master_config().get_all_profile_names()]
+              profile in
+              get_master_config().get_all_profile_names()]
         )
 
         menu = Menu(
@@ -486,6 +530,7 @@ class GSMTray(threading.Thread):
         if self.icon:
             self.icon.stop()
 
+
 gsm_tray = GSMTray()
 
 
@@ -540,13 +585,13 @@ def cleanup():
         obs.disconnect_from_obs()
         if get_config().obs.close_obs:
             close_obs()
-            
+
         if texthooking_page.websocket_server_threads:
             for thread in texthooking_page.websocket_server_threads:
                 if thread and isinstance(thread, threading.Thread) and thread.is_alive():
                     thread.stop_server()
                     thread.join()
-        
+
         proc: Popen
         for proc in procs_to_close:
             try:
@@ -568,7 +613,8 @@ def cleanup():
                 if os.path.exists(video):
                     os.remove(video)
             except Exception as e:
-                logger.error(f"Error removing temporary video file {video}: {e}")
+                logger.error(
+                    f"Error removing temporary video file {video}: {e}")
 
         settings_window.window.destroy()
         # time.sleep(5)
@@ -668,6 +714,9 @@ def async_loop():
         await register_scene_switcher_callback()
         await check_obs_folder_is_correct()
         vad_processor.init()
+        OverlayThread().start()
+
+        # Keep loop alive
         # if is_beangate:
         # await run_test_code()
 
@@ -713,8 +762,8 @@ async def run_test_code():
         if boxes:
             await texthooking_page.send_word_coordinates_to_overlay(boxes)
         await asyncio.sleep(2)
-        
-        
+
+
 async def check_if_script_is_running():
     """Check if the script is already running and kill it if so."""
     if os.path.exists(os.path.join(get_app_directory(), "current_pid.txt")):
@@ -722,14 +771,15 @@ async def check_if_script_is_running():
             pid = int(f.read().strip())
             if psutil.pid_exists(pid) and 'python' in psutil.Process(pid).name().lower():
                 logger.info(f"Script is already running with PID: {pid}")
-                psutil.Process(pid).terminate()  # Attempt to terminate the existing process
+                # Attempt to terminate the existing process
+                psutil.Process(pid).terminate()
                 logger.info("Sent SIGTERM to the existing process.")
                 notification.send_error_notification(
                     "Script was already running. Terminating the existing process.")
                 return True
     return False
-    
-    
+
+
 async def log_current_pid():
     """Log the current process ID."""
     current_pid = os.getpid()
@@ -748,17 +798,17 @@ async def async_main(reloading=False):
         initialize_async()
         observer = Observer()
         observer.schedule(VideoToAudioHandler(),
-                        get_config().paths.folder_to_watch, recursive=False)
+                          get_config().paths.folder_to_watch, recursive=False)
         observer.start()
         if is_windows():
             register_hotkeys()
-            
+
         run_new_thread(initialize_text_monitor)
         run_new_thread(run_text_hooker_page)
         run_new_thread(async_loop).join()
-        
+
         logger.info("Initialization complete. Happy Mining! がんばれ！")
-        
+
         # await check_if_script_is_running()
         # await log_current_pid()
 
@@ -797,10 +847,9 @@ def main():
         handle_error_in_initialization(e)
 
 
-
 if __name__ == "__main__":
     logger.info("Starting GSM")
     try:
         asyncio.run(async_main())
     except Exception as e:
-        handle_error_in_initialization(e)
+        handle_error_in_initialization(e)

GameSentenceMiner/locales/en_us.json

@@ -268,6 +268,14 @@
           "use_cpu_for_inference": {
             "label": "Force CPU:",
             "tooltip": "Even if CUDA is installed, use CPU for Whisper"
+          },
+          "use_tts_as_fallback": {
+            "label": "Use TTS as Fallback:",
+            "tooltip": "Use Text-to-Speech as a fallback when no audio is found."
+          },
+          "tts_url": {
+            "label": "TTS URL:",
+            "tooltip": "URL for the Text-to-Speech service. Use $s as a placeholder for the text."
           }
         },
         "features": {
@@ -576,6 +584,14 @@
           "overlay_engine": {
             "label": "Overlay Engine:",
             "tooltip": "Select the OCR engine for the overlay. If you use lens, and are on windows, it will use OneOCR to optimize the scan."
+          },
+          "periodic": {
+            "label": "Periodic Capture:",
+            "tooltip": "Enable periodic capture of the screen for Overlay. Note, you still need text flowing into GSM for mining to work."
+          },
+          "periodic_interval": {
+            "label": "Capture Interval (Seconds):",
+            "tooltip": "Interval in seconds for periodic screen capture."
           }
         },
         "wip": {

GameSentenceMiner/locales/ja_jp.json

@@ -267,6 +267,14 @@
           "use_cpu_for_inference": {
             "label": "CPU強制使用:",
             "tooltip": "CUDAがインストールされていてもWhisperでCPUを使用します"
+          },
+          "use_tts_as_fallback": {
+            "label": "TTSをフォールバックとして使用:",
+            "tooltip": "音声が見つからない場合にテキスト読み上げをフォールバックとして使用します。"
+          },
+          "tts_url": {
+            "label": "TTS URL:",
+            "tooltip": "テキスト読み上げサービスのURL。テキストのプレースホルダーとして$sを使用します。"
           }
         },
         "features": {
@@ -575,6 +583,14 @@
           "overlay_engine": {
             "label": "オーバーレイエンジン:",
             "tooltip": "オーバーレイのOCRエンジンを選択します。Lensを使用していてWindowsの場合、スキャンを最適化するためにOneOCRを使用します。"
+          },
+          "periodic": {
+            "label": "定期キャプチャ:",
+            "tooltip": "OCR処理のために画面を定期的にキャプチャするかどうか。"
+          },
+          "periodic_interval": {
+            "label": "キャプチャ間隔（秒）:",
+            "tooltip": "定期的な画面キャプチャの間隔（秒単位）。"
           }
         },
         "wip": {

GameSentenceMiner/locales/zh_cn.json

@@ -268,6 +268,14 @@
           "use_cpu_for_inference": {
             "label": "强制使用 CPU:",
             "tooltip": "即使已安装 CUDA，也强制使用 CPU 运行 Whisper"
+          },
+          "use_tts_as_fallback": {
+            "label": "使用 TTS 作为后备:",
+            "tooltip": "在未找到音频时使用文本转语音作为后备。"
+          },
+          "tts_url": {
+            "label": "TTS URL:",
+            "tooltip": "文本转语音服务的 URL。使用 $s 作为文本的占位符。"
           }
         },
         "features": {
@@ -564,6 +572,14 @@
           "overlay_engine": {
             "label": "覆盖层引擎:",
             "tooltip": "为覆盖层选择 OCR 引擎。如果您使用的是 lens，并且在 windows 上，它将使用 OneOCR 来优化扫描。"
+          },
+          "periodic": {
+            "label": "定期捕获:",
+            "tooltip": "启用定期屏幕捕获以进行 OCR 处理。"
+          },
+          "periodic_interval": {
+            "label": "捕获间隔（秒）:",
+            "tooltip": "定期屏幕捕获的时间间隔（秒）。"
           }
         },
         "wip": {

GameSentenceMiner/util/configuration.py

@@ -558,6 +558,8 @@ class VAD:
     trim_beginning: bool = False
     beginning_offset: float = -0.25
     add_audio_on_no_results: bool = False
+    use_tts_as_fallback: bool = False
+    tts_url: str = 'http://127.0.0.1:5050/?term=$s'
     cut_and_splice_segments: bool = False
     splice_padding: float = 0.1
     use_cpu_for_inference: bool = False
@@ -642,6 +644,8 @@ class Overlay:
     websocket_port: int = 55499
     engine: str = OverlayEngine.LENS.value
     monitor_to_capture: int = 0
+    periodic: bool = False
+    periodic_interval: float = 1.0
 
     def __post_init__(self):
         if self.monitor_to_capture == -1:

GameSentenceMiner/util/get_overlay_coords.py

@@ -4,6 +4,7 @@ import base64
 import json
 import math
 import os
+import threading
 import time
 from PIL import Image
 from typing import Dict, Any, List, Tuple
@@ -16,6 +17,7 @@ from GameSentenceMiner.util.configuration import OverlayEngine, get_config, is_w
 from GameSentenceMiner.util.electron_config import get_ocr_language
 from GameSentenceMiner.obs import get_screenshot_PIL
 from GameSentenceMiner.web.texthooking_page import send_word_coordinates_to_overlay
+from GameSentenceMiner.web.gsm_websocket import overlay_server_thread
 
 # def align_and_correct(ocr_json, reference_text):
 #     logger.info(f"Starting align_and_correct with reference_text: '{reference_text}'")
@@ -80,6 +82,32 @@ try:
     import mss
 except ImportError:
     mss = None
+    
+class OverlayThread(threading.Thread):
+    """
+    A thread to run the overlay processing loop.
+    This is a simple wrapper around asyncio to run the overlay processing
+    in a separate thread.
+    """
+    def __init__(self):
+        super().__init__()
+        self.overlay_processor = OverlayProcessor()
+        self.loop = asyncio.new_event_loop()
+        self.daemon = True  # Ensure thread exits when main program exits
+
+    def run(self):
+        """Runs the overlay processing loop."""
+        asyncio.set_event_loop(self.loop)
+        self.loop.run_until_complete(self.overlay_loop())
+
+    async def overlay_loop(self):
+        """Main loop to periodically process and send overlay data."""
+        while True:
+            if get_config().overlay.periodic and overlay_server_thread.has_clients():
+                await self.overlay_processor.find_box_and_send_to_overlay('')
+                await asyncio.sleep(get_config().overlay.periodic_interval)  # Adjust the interval as needed
+            else:
+                await asyncio.sleep(3)  # Sleep briefly when not active
 
 class OverlayProcessor:
     """
@@ -254,7 +282,7 @@ class OverlayProcessor:
                 return_coords=True,
                 multiple_crop_coords=True,
                 return_one_box=False,
-                furigana_filter_sensitivity=None # Disable furigana filtering
+                furigana_filter_sensitivity=None, # Disable furigana filtering
             )
 
             # 3. Create a composite image with only the detected text regions

GameSentenceMiner/vad.py

@@ -1,6 +1,7 @@
 import tempfile
 import time
 import warnings
+import requests
 from abc import abstractmethod, ABC
 
 from GameSentenceMiner.util import configuration, ffmpeg
@@ -37,19 +38,7 @@ class VADSystem:
             if not result.success and get_config().vad.backup_vad_model != configuration.OFF:
                 logger.info("No voice activity detected, using backup VAD model.")
                 result = self._do_vad_processing(get_config().vad.backup_vad_model, input_audio, output_audio, game_line)
-            if not result.success:
-                if get_config().vad.add_audio_on_no_results:
-                    logger.info("No voice activity detected, using full audio.")
-                    result.output_audio = input_audio
-                else:
-                    logger.info("No voice activity detected.")
-                    return result
-            else:
-                logger.info(result.trim_successful_string())
             return result
-        else:
-            return VADResult(True, 0, get_audio_length(input_audio), "OFF", [], input_audio)
-
 
     def _do_vad_processing(self, model, input_audio, output_audio, game_line):
         match model:

GameSentenceMiner/web/templates/stats.html

@@ -361,7 +361,7 @@
                                 AFK Timer (seconds)
                             </label>
                             <input type="number" id="afkTimer" name="afk_timer_seconds"
-                                style="width: 90%; padding: 10px; border: 1px solid var(--border-color); border-radius: 5px; background: var(--bg-tertiary); color: var(--text-primary); font-size: 14px;"
+                                style="padding: 10px; border: 1px solid var(--border-color); border-radius: 5px; background: var(--bg-tertiary); color: var(--text-primary); font-size: 14px;"
                                 placeholder="120">
                             <small
                                 style="color: var(--text-tertiary); font-size: 12px; margin-top: 4px; display: block;">
@@ -375,7 +375,7 @@
                                 Session Gap (seconds)
                             </label>
                             <input type="number" id="sessionGap" name="session_gap_seconds"
-                                style="width: 90%; padding: 10px; border: 1px solid var(--border-color); border-radius: 5px; background: var(--bg-tertiary); color: var(--text-primary); font-size: 14px;"
+                                style="padding: 10px; border: 1px solid var(--border-color); border-radius: 5px; background: var(--bg-tertiary); color: var(--text-primary); font-size: 14px;"
                                 placeholder="3600">
                             <small
                                 style="color: var(--text-tertiary); font-size: 12px; margin-top: 4px; display: block;">
@@ -405,7 +405,7 @@
                             </label>
                             <input type="number" id="streakRequirement" name="streak_requirement_hours" min="0.01"
                                 max="24" step="0.01"
-                                style="width: 90%; padding: 10px; border: 1px solid var(--border-color); border-radius: 5px; background: var(--bg-tertiary); color: var(--text-primary); font-size: 14px;"
+                                style="padding: 10px; border: 1px solid var(--border-color); border-radius: 5px; background: var(--bg-tertiary); color: var(--text-primary); font-size: 14px;"
                                 placeholder="1.0">
                             <small
                                 style="color: var(--text-tertiary); font-size: 12px; margin-top: 4px; display: block;">
@@ -430,7 +430,7 @@
                                 </label>
                                 <input type="number" id="readingHoursTarget" name="reading_hours_target" min="1"
                                     max="10000"
-                                    style="width: 90%; padding: 10px; border: 1px solid var(--border-color); border-radius: 5px; background: var(--bg-tertiary); color: var(--text-primary); font-size: 14px;"
+                                    style="padding: 10px; border: 1px solid var(--border-color); border-radius: 5px; background: var(--bg-tertiary); color: var(--text-primary); font-size: 14px;"
                                     placeholder="1500">
                                 <small
                                     style="color: var(--text-tertiary); font-size: 12px; margin-top: 4px; display: block;">
@@ -446,7 +446,7 @@
                                 </label>
                                 <input type="number" id="characterCountTarget" name="character_count_target" min="1000"
                                     max="1000000000"
-                                    style="width: 90%; padding: 10px; border: 1px solid var(--border-color); border-radius: 5px; background: var(--bg-tertiary); color: var(--text-primary); font-size: 14px;"
+                                    style="padding: 10px; border: 1px solid var(--border-color); border-radius: 5px; background: var(--bg-tertiary); color: var(--text-primary); font-size: 14px;"
                                     placeholder="25000000">
                                 <small
                                     style="color: var(--text-tertiary); font-size: 12px; margin-top: 4px; display: block;">

{gamesentenceminer-2.16.8.dist-info → gamesentenceminer-2.16.9.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: GameSentenceMiner
-Version: 2.16.8
+Version: 2.16.9
 Summary: A tool for mining sentences from games. Update: Overlay?
 Author-email: Beangate <bpwhelan95@gmail.com>
 License: MIT License

{gamesentenceminer-2.16.8.dist-info → gamesentenceminer-2.16.9.dist-info}/RECORD

@@ -1,10 +1,10 @@
 GameSentenceMiner/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 GameSentenceMiner/anki.py,sha256=Qq03nxYCA0bXS8IR1vnEB9cv2vxo6Ruy-UuiojC4ad0,26518
-GameSentenceMiner/config_gui.py,sha256=o_7zNONJPtbg6k-lRIpT2vUtxnEhBUljriRCb0zRuuI,143350
+GameSentenceMiner/config_gui.py,sha256=iOsi3IVati6JHCnoCIzLAxlrbZt4j4pL5KNFzSq4gJo,146121
 GameSentenceMiner/gametext.py,sha256=fgBgLchezpauWELE9Y5G3kVCLfAneD0X4lJFoI3FYbs,10351
-GameSentenceMiner/gsm.py,sha256=CcXGZI8K7cYB9FJfCRaOujlWDk08sQGQXhTiMewMhFg,31889
+GameSentenceMiner/gsm.py,sha256=bxuP6gez7Yw-iNxNRMHy6qtDUQy0cNmmwJpnv1crL4Y,33627
 GameSentenceMiner/obs.py,sha256=EyAYhaLvMjoeC-3j7fuvkqZN5logFFanPfb8Wn1C6m0,27296
-GameSentenceMiner/vad.py,sha256=MdeovQGrwIlNdIa8wKWBBo0qSBp-Ay1-hZRe0YhZBSU,20257
+GameSentenceMiner/vad.py,sha256=klkxPA1pNbziZWG1MGjWVRRmOIt9UwlcB8ZV-lSnHsQ,19736
 GameSentenceMiner/ai/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 GameSentenceMiner/ai/ai_prompting.py,sha256=41xdBzE88Jlt12A0D-T_cMfLO5j6MSxfniOptpwNZm0,24068
 GameSentenceMiner/assets/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -15,9 +15,9 @@ GameSentenceMiner/assets/icon32.png,sha256=Kww0hU_qke9_22wBuO_Nq0Dv2SfnOLwMhCyGg
 GameSentenceMiner/assets/icon512.png,sha256=HxUj2GHjyQsk8NV433256UxU9phPhtjCY-YB_7W4sqs,192487
 GameSentenceMiner/assets/icon64.png,sha256=N8xgdZXvhqVQP9QUK3wX5iqxX9LxHljD7c-Bmgim6tM,9301
 GameSentenceMiner/assets/pickaxe.png,sha256=VfIGyXyIZdzEnVcc4PmG3wszPMO1W4KCT7Q_nFK6eSE,1403829
-GameSentenceMiner/locales/en_us.json,sha256=VZ2RblOdRLUaTyofcJtc-urmW88vFL1G7DMG7wKbw9k,27184
-GameSentenceMiner/locales/ja_jp.json,sha256=X00lVjbxEhH3NVBxFMqqWMMh388DDGUvDECXGyf8T0k,29014
-GameSentenceMiner/locales/zh_cn.json,sha256=EYbNwLffCyrBw9AEiIObo6mZw9G_nsEkRDiN1QrFhoc,25259
+GameSentenceMiner/locales/en_us.json,sha256=pyh8X6BHVny9Rk3fTBcW9wgbyysvpgWT2cebFAzIyv4,27917
+GameSentenceMiner/locales/ja_jp.json,sha256=zqbZM04BV0qWR7Al_vdsL8b2hNq_B6oq2msvJc1AMOA,29854
+GameSentenceMiner/locales/zh_cn.json,sha256=QLkCZF6zomPGT52UpS8x4ntW3YdeJyfvQTqyZgqdcFk,25914
 GameSentenceMiner/ocr/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 GameSentenceMiner/ocr/gsm_ocr_config.py,sha256=Ov04c-nKzh3sADxO-5JyZWVe4DlrHM9edM9tc7-97Jo,5970
 GameSentenceMiner/ocr/ocrconfig.py,sha256=_tY8mjnzHMJrLS8E5pHqYXZjMuLoGKYgJwdhYgN-ny4,6466
@@ -37,11 +37,11 @@ GameSentenceMiner/tools/furigana_filter_preview.py,sha256=BXv7FChPEJW_VeG5XYt6su
 GameSentenceMiner/tools/ss_selector.py,sha256=cbjMxiKOCuOfbRvLR_PCRlykBrGtm1LXd6u5czPqkmc,4793
 GameSentenceMiner/tools/window_transparency.py,sha256=GtbxbmZg0-UYPXhfHff-7IKZyY2DKe4B9GdyovfmpeM,8166
 GameSentenceMiner/util/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-GameSentenceMiner/util/configuration.py,sha256=U59JYqUfq5ty_sLMNs-LVrWlc6wXB5-fCM8rstb9cR8,42291
+GameSentenceMiner/util/configuration.py,sha256=qATOwZahVQNP8-ZnWiAKuR7UJLW25QNDmFKYNBYzkmE,42443
 GameSentenceMiner/util/db.py,sha256=B2Qwg7i0Qn_yxov-NhcT9QdFkF218Cqea_V7ZPzYBzM,21365
 GameSentenceMiner/util/electron_config.py,sha256=KfeJToeFFVw0IR5MKa-gBzpzaGrU-lyJbR9z-sDEHYU,8767
 GameSentenceMiner/util/ffmpeg.py,sha256=g3v1aJnv3qWekDnO0FdYozB-MG9di4WUvPA3NyXY9Ws,28998
-GameSentenceMiner/util/get_overlay_coords.py,sha256=TEMxhrBE8302WQj2K5q7SLKUhgNH_J-RjHpkUTQJdcU,17577
+GameSentenceMiner/util/get_overlay_coords.py,sha256=LDm32E0MG-4vqT7Vv4hhbP9_lK-TJahTkyWes0TPaeM,18753
 GameSentenceMiner/util/gsm_utils.py,sha256=Piwv88Q9av2LBeN7M6QDi0Mp0_R2lNbkcI6ekK5hd2o,11851
 GameSentenceMiner/util/model.py,sha256=R-_RYTYLSDNgBoVTPuPBcIHeOznIqi_vBzQ7VQ20WYk,6727
 GameSentenceMiner/util/notification.py,sha256=YBhf_mSo_i3cjBz-pmeTPx3wchKiG9BK2VBdZSa2prQ,4597
@@ -85,14 +85,14 @@ GameSentenceMiner/web/templates/anki_stats.html,sha256=FdIMl-kY0-3as9Wn0i-wKIIkl
 GameSentenceMiner/web/templates/database.html,sha256=Gd54rgZmfcV7ufoJ69COeMncs5Q5u-rSJcsIvROVCEo,13732
 GameSentenceMiner/web/templates/index.html,sha256=7ChQ1j602MOiYU95wXAKP_Ezsh_JgdlGz2uXIzS2j0g,227894
 GameSentenceMiner/web/templates/search.html,sha256=nao3M_hAbm5ftzThi91NtQ3ZiINMPUNx4ngFmqLnLQ4,4060
-GameSentenceMiner/web/templates/stats.html,sha256=RRhQ8q0Gimt9gO2aa4iMguNYbijjEhHPJdCpXkV837s,31596
+GameSentenceMiner/web/templates/stats.html,sha256=BL7HQLs62RbPbo5Hs8BgdH9CsrGLH583JSn0Uz3PK5Y,31536
 GameSentenceMiner/web/templates/utility.html,sha256=KtqnZUMAYs5XsEdC9Tlsd40NKAVic0mu6sh-ReMDJpU,16940
 GameSentenceMiner/web/templates/components/navigation.html,sha256=6y9PvM3nh8LY6JWrZb6zVOm0vqkBLDc6d3gB9X5lT_w,1055
 GameSentenceMiner/web/templates/components/theme-styles.html,sha256=hiq3zdJljpRjQO1iUA7gfFKwXebltG-IWW-gnKS4GHA,3439
 GameSentenceMiner/wip/__init___.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-gamesentenceminer-2.16.8.dist-info/licenses/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
-gamesentenceminer-2.16.8.dist-info/METADATA,sha256=RAXeKbYDIbfz9SU0p_R5Vrgjd-XnyiJ1x3H-WFsns-0,7348
-gamesentenceminer-2.16.8.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-gamesentenceminer-2.16.8.dist-info/entry_points.txt,sha256=2APEP25DbfjSxGeHtwBstMH8mulVhLkqF_b9bqzU6vQ,65
-gamesentenceminer-2.16.8.dist-info/top_level.txt,sha256=V1hUY6xVSyUEohb0uDoN4UIE6rUZ_JYx8yMyPGX4PgQ,18
-gamesentenceminer-2.16.8.dist-info/RECORD,,
+gamesentenceminer-2.16.9.dist-info/licenses/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
+gamesentenceminer-2.16.9.dist-info/METADATA,sha256=e_346ixgOw2sn9kslL0d_bAorQZbwYKGoHZNpZWdE7M,7348
+gamesentenceminer-2.16.9.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+gamesentenceminer-2.16.9.dist-info/entry_points.txt,sha256=2APEP25DbfjSxGeHtwBstMH8mulVhLkqF_b9bqzU6vQ,65
+gamesentenceminer-2.16.9.dist-info/top_level.txt,sha256=V1hUY6xVSyUEohb0uDoN4UIE6rUZ_JYx8yMyPGX4PgQ,18
+gamesentenceminer-2.16.9.dist-info/RECORD,,