GameSentenceMiner 2.3.7__py3-none-any.whl → 2.4.0__py3-none-any.whl

GameSentenceMiner/anki.py

@@ -1,5 +1,4 @@
 import base64
-import queue
 import subprocess
 import threading
 import time
@@ -11,7 +10,7 @@ from GameSentenceMiner import obs, util, notification, ffmpeg, gametext
 
 from GameSentenceMiner.configuration import *
 from GameSentenceMiner.configuration import get_config
-from GameSentenceMiner.gametext import get_last_two_sentences
+from GameSentenceMiner.gametext import get_text_event
 from GameSentenceMiner.obs import get_current_game
 from GameSentenceMiner.util import remove_html_tags
 
@@ -26,7 +25,7 @@ card_queue = []
 
 
 def update_anki_card(last_note, note=None, audio_path='', video_path='', tango='', reuse_audio=False,
-                     should_update_audio=True, ss_time=0):
+                     should_update_audio=True, ss_time=0, game_line=None):
     global audio_in_anki, screenshot_in_anki, prev_screenshot_in_anki
     update_audio = should_update_audio and (get_config().anki.sentence_audio_field and not
     last_note['fields'][get_config().anki.sentence_audio_field][
@@ -44,12 +43,10 @@ def update_anki_card(last_note, note=None, audio_path='', video_path='', tango='
             if get_config().paths.remove_screenshot:
                 os.remove(screenshot)
         if get_config().anki.previous_image_field:
-            _, previous_sentence = get_last_two_sentences(last_note)
-            prev_screenshot = ffmpeg.get_screenshot(video_path, ffmpeg.get_screenshot_time(video_path, gametext.get_time_of_line(previous_sentence)))
+            prev_screenshot = ffmpeg.get_screenshot(video_path, ffmpeg.get_screenshot_time(video_path, game_line.prev))
             prev_screenshot_in_anki = store_media_file(prev_screenshot)
             if get_config().paths.remove_screenshot:
                 os.remove(prev_screenshot)
-        util.set_last_mined_line(get_sentence(last_note))
     audio_html = f"[sound:{audio_in_anki}]"
     image_html = f"<img src=\"{screenshot_in_anki}\">"
     prev_screenshot_html = f"<img src=\"{prev_screenshot_in_anki}\">"
@@ -122,7 +119,7 @@ def get_initial_card_info(last_note, selected_lines):
     note = {'id': last_note['noteId'], 'fields': {}}
     if not last_note:
         return note
-    current_line, previous_line = get_last_two_sentences(last_note)
+    game_line = get_text_event(last_note)
 
     if get_config().audio.mining_from_history_grab_all_audio and get_config().anki.multi_overwrites_sentence:
         lines = gametext.get_line_and_future_lines(last_note)
@@ -130,13 +127,13 @@ def get_initial_card_info(last_note, selected_lines):
             note['fields'][get_config().anki.sentence_field] = "".join(lines)
 
     if selected_lines and get_config().anki.multi_overwrites_sentence:
-        note['fields'][get_config().anki.sentence_field] = "".join(selected_lines)
+        note['fields'][get_config().anki.sentence_field] = "".join([line.text for line in selected_lines])
 
-    logger.debug(
-        f"Adding Previous Sentence: {get_config().anki.previous_sentence_field and previous_line and not last_note['fields'][get_config().anki.previous_sentence_field]['value']}")
-    if get_config().anki.previous_sentence_field and previous_line and not \
+    if get_config().anki.previous_sentence_field and game_line.prev and not \
             last_note['fields'][get_config().anki.previous_sentence_field]['value']:
-        note['fields'][get_config().anki.previous_sentence_field] = previous_line
+        logger.debug(
+            f"Adding Previous Sentence: {get_config().anki.previous_sentence_field and game_line.prev.text and not last_note['fields'][get_config().anki.previous_sentence_field]['value']}")
+        note['fields'][get_config().anki.previous_sentence_field] = game_line.prev.text
     return note
 
 
@@ -227,7 +224,8 @@ def update_new_card():
     if get_config().obs.get_game_from_scene:
         obs.update_current_game()
     if use_prev_audio:
-        update_anki_card(last_card, note=get_initial_card_info(last_card, []), reuse_audio=True)
+        with util.lock:
+            update_anki_card(last_card, note=get_initial_card_info(last_card, []), reuse_audio=True)
     else:
         logger.info("New card(s) detected! Added to Processing Queue!")
         card_queue.append(last_card)

GameSentenceMiner/config_gui.py

@@ -51,6 +51,7 @@ class HoverInfoWidget:
 class ConfigApp:
     def __init__(self, root):
         self.window = root
+        self.on_exit = None
         # self.window = ttk.Window(themename='darkly')
         self.window.title('GameSentenceMiner Configuration')
         self.window.protocol("WM_DELETE_WINDOW", self.hide)
@@ -82,10 +83,6 @@ class ConfigApp:
     def add_save_hook(self, func):
         on_save.append(func)
 
-    def add_exit_hook(self, func):
-        global exit_func
-        exit_func = func
-
     def show(self):
         obs.update_current_game()
         self.reload_settings()
@@ -103,7 +100,7 @@ class ConfigApp:
         if update_available:
             messagebox.showinfo("Update", "GSM Will Copy the Update Command to your clipboard, please run it in a terminal.")
             pyperclip.copy("pip install --upgrade GameSentenceMiner")
-            exit_func(None, None)
+            self.on_exit(None, None)
         else:
             messagebox.showinfo("No Update Found", "No update found.")
 

GameSentenceMiner/configuration.py

@@ -7,6 +7,7 @@ from logging.handlers import RotatingFileHandler
 from os.path import expanduser
 from sys import platform
 from typing import List, Dict
+import sys
 
 import toml
 from dataclasses_json import dataclass_json
@@ -370,11 +371,14 @@ def switch_profile_and_save(profile_name):
     return config_instance.get_config()
 
 
+sys.stdout.reconfigure(encoding='utf-8')
+sys.stderr.reconfigure(encoding='utf-8')
+
 logger = logging.getLogger("GameSentenceMiner")
 logger.setLevel(logging.DEBUG)  # Set the base level to DEBUG so that all messages are captured
 
 # Create console handler with level INFO
-console_handler = logging.StreamHandler()
+console_handler = logging.StreamHandler(sys.stdout)
 console_handler.setLevel(logging.INFO)
 
 # Create rotating file handler with level DEBUG

GameSentenceMiner/ffmpeg.py

@@ -5,6 +5,8 @@ import time
 from GameSentenceMiner import obs, util, configuration
 from GameSentenceMiner.configuration import *
 from GameSentenceMiner.util import *
+from GameSentenceMiner.gametext import initial_time
+
 
 def get_ffmpeg_path():
     return os.path.join(get_app_directory(), "ffmpeg", "ffmpeg.exe") if util.is_windows() else "ffmpeg"
@@ -47,7 +49,12 @@ def get_screenshot(video_file, time_from_end):
     return output_image
 
 
-def get_screenshot_time(video_path, line_time):
+def get_screenshot_time(video_path, game_line):
+    if game_line:
+        line_time = game_line.time
+    else:
+        line_time = initial_time
+
     file_length = get_video_duration(video_path)
     file_mod_time = get_file_modification_time(video_path)
 
@@ -115,7 +122,7 @@ def get_audio_codec(video_path):
         return None
 
 
-def get_audio_and_trim(video_path, line_time, next_line_time):
+def get_audio_and_trim(video_path, game_line, next_line_time):
     supported_formats = {
         'opus': 'libopus',
         'mp3': 'libmp3lame',
@@ -149,7 +156,7 @@ def get_audio_and_trim(video_path, line_time, next_line_time):
 
     subprocess.run(command)
 
-    return trim_audio_based_on_last_line(untrimmed_audio, video_path, line_time, next_line_time)
+    return trim_audio_based_on_last_line(untrimmed_audio, video_path, game_line, next_line_time)
 
 
 def get_video_duration(file_path):
@@ -167,12 +174,12 @@ def get_video_duration(file_path):
     return float(duration_info["format"]["duration"])  # Return the duration in seconds
 
 
-def trim_audio_based_on_last_line(untrimmed_audio, video_path, line_time, next_line):
+def trim_audio_based_on_last_line(untrimmed_audio, video_path, game_line, next_line):
     trimmed_audio = tempfile.NamedTemporaryFile(dir=configuration.get_temporary_directory(),
                                                 suffix=f".{get_config().audio.extension}").name
     file_mod_time = get_file_modification_time(video_path)
     file_length = get_video_duration(video_path)
-    time_delta = file_mod_time - line_time
+    time_delta = file_mod_time - game_line.time
     # Convert time_delta to FFmpeg-friendly format (HH:MM:SS.milliseconds)
     total_seconds = file_length - time_delta.total_seconds() + get_config().audio.beginning_offset
     if total_seconds < 0 or total_seconds >= file_length:
@@ -186,8 +193,8 @@ def trim_audio_based_on_last_line(untrimmed_audio, video_path, line_time, next_l
     ffmpeg_command = ffmpeg_base_command_list + [
         "-i", untrimmed_audio,
         "-ss", start_trim_time]
-    if next_line and next_line > line_time and not get_config().audio.mining_from_history_grab_all_audio:
-        end_total_seconds = total_seconds + (next_line - line_time).total_seconds() + 1
+    if next_line and next_line > game_line.time and not get_config().audio.mining_from_history_grab_all_audio:
+        end_total_seconds = total_seconds + (next_line - game_line.time).total_seconds() + 1
         hours, remainder = divmod(end_total_seconds, 3600)
         minutes, seconds = divmod(remainder, 60)
         end_trim_time = "{:02}:{:02}:{:06.3f}".format(int(hours), int(minutes), seconds)

GameSentenceMiner/gametext.py

@@ -2,7 +2,6 @@ import asyncio
 import re
 import threading
 import time
-from collections import OrderedDict
 from datetime import datetime
 from typing import Callable
 
@@ -21,10 +20,61 @@ current_line = ''
 current_line_after_regex = ''
 current_line_time = datetime.now()
 
-line_history = OrderedDict()
 reconnecting = False
 multi_mine_event_bus: Callable[[str, datetime], None] = None
 
+@dataclass
+class GameLine:
+    text: str
+    time: datetime
+    prev: 'GameLine'
+    next: 'GameLine'
+
+    def get_previous_time(self):
+        if self.prev:
+            return self.prev.time
+        return initial_time
+
+    def get_next_time(self):
+        if self.next:
+            return self.next.time
+        return 0
+
+@dataclass
+class GameText:
+    values: list[GameLine]
+
+    def __init__(self):
+        self.values = []
+
+    def __getitem__(self, key):
+        return self.values[key]
+
+    def get_time(self, line_text: str, occurrence: int = -1) -> datetime:
+        matches = [line for line in self.values if line.text == line_text]
+        if matches:
+            return matches[occurrence].time  # Default to latest
+        return initial_time
+
+    def get_event(self, line_text: str, occurrence: int = -1) -> GameLine | None:
+        matches = [line for line in self.values if line.text == line_text]
+        if matches:
+            return matches[occurrence]
+        return None
+
+    def add_line(self, line_text):
+        new_line = GameLine(line_text, datetime.now(), self.values[-1] if self.values else None, None)
+        if self.values:
+            self.values[-1].next = new_line
+        self.values.append(new_line)
+
+    def has_line(self, line_text) -> bool:
+        for game_line in self.values:
+            if game_line.text == line_text:
+                return True
+        return False
+
+line_history = GameText()
 
 class ClipboardMonitor(threading.Thread):
 
@@ -79,17 +129,16 @@ def handle_new_text_event(current_clipboard):
         current_line_after_regex = re.sub(get_config().general.texthook_replacement_regex, '', current_line)
     else:
         current_line_after_regex = current_line
+    logger.info(f"Line Received: {current_line_after_regex}")
     current_line_time = datetime.now()
-    line_history[current_line_after_regex] = current_line_time
-    multi_mine_event_bus(current_line_after_regex, current_line_time)
-    logger.debug(f"New Line: {current_clipboard}")
+    line_history.add_line(current_line_after_regex)
+    multi_mine_event_bus(line_history[-1])
 
 
 def reset_line_hotkey_pressed():
     global current_line_time
     logger.info("LINE RESET HOTKEY PRESSED")
     current_line_time = datetime.now()
-    line_history[current_line_after_regex] = current_line_time
     util.set_last_mined_line("")
 
 
@@ -107,72 +156,91 @@ def start_text_monitor(send_to_mine_event_bus):
     text_thread.start()
 
 
-def get_line_timing(last_note):
-    def similar(a, b):
-        return SequenceMatcher(None, a, b).ratio()
+# def get_line_timing(last_note):
+#     def similar(a, b):
+#         return SequenceMatcher(None, a, b).ratio()
+#
+#     if not last_note:
+#         return current_line_time, 0
+#
+#     lines = line_history.values
+#
+#     line_time = current_line_time
+#     next_line = 0
+#     prev_clip_time = 0
+#
+#     try:
+#         sentence = last_note['fields'][get_config().anki.sentence_field]['value']
+#         if sentence:
+#             for line in reversed(lines):
+#                 similarity = similar(remove_html_tags(sentence), line.text)
+#                 if similarity >= 0.60 or line.text in remove_html_tags(sentence):  # 80% similarity threshold
+#                     line_time = line.time
+#                     next_line = prev_clip_time
+#                     break
+#                 prev_clip_time = line.time
+#     except Exception as e:
+#         logger.error(f"Using Default clipboard/websocket timing - reason: {e}")
+#
+#     return line_time, next_line
+#
+#
+# def get_last_two_sentences(last_note) -> (str, str):
+#     def similar(a, b):
+#         return SequenceMatcher(None, a, b).ratio()
+#
+#     lines = line_history.values
+#
+#     if not last_note:
+#         return lines[-1].text if lines else '', lines[-2].text if len(lines) > 1 else ''
+#
+#     sentence = last_note['fields'][get_config().anki.sentence_field]['value']
+#     if not sentence:
+#         return lines[-1].text if lines else '', lines[-2].text if len(lines) > 1 else ''
+#
+#     current, previous = "", ""
+#     found = False
+#
+#     for line in reversed(lines):
+#         similarity = similar(remove_html_tags(sentence), line.text)
+#         logger.debug(f"Comparing: {remove_html_tags(sentence)} with {line.text} - Similarity: {similarity}")
+#         if found:
+#             previous = line.text
+#             break
+#         if similarity >= 0.60 or line.text in remove_html_tags(sentence):
+#             found = True
+#             current = line.text
+#
+#     if not current or not previous:
+#         logger.debug("Couldn't find lines in history, using last two lines")
+#         return lines[-1].text if lines else '', lines[-2].text if len(lines) > 1 else ''
+#
+#     return current, previous
+
+def similar(a, b):
+    return SequenceMatcher(None, a, b).ratio()
+
+def get_text_event(last_note) -> GameLine:
+    lines = line_history.values
 
     if not last_note:
-        return current_line_time, 0
-
-    line_time = current_line_time
-    next_line = 0
-    prev_clip_time = 0
-
-    try:
-        sentence = last_note['fields'][get_config().anki.sentence_field]['value']
-        if sentence:
-            for i, (line, clip_time) in enumerate(reversed(line_history.items())):
-                similarity = similar(remove_html_tags(sentence), line)
-                if similarity >= 0.60 or line in remove_html_tags(sentence):  # 80% similarity threshold
-                    line_time = clip_time
-                    next_line = prev_clip_time
-                    break
-                prev_clip_time = clip_time
-    except Exception as e:
-        logger.error(f"Using Default clipboard/websocket timing - reason: {e}")
-
-    return line_time, next_line
-
-
-def get_last_two_sentences(last_note):
-    def similar(a, b):
-        return SequenceMatcher(None, a, b).ratio()
-    lines = list(line_history.items())
-
-    if not last_note:
-        return lines[-1][0] if lines else '', lines[-2][0] if len(lines) > 1 else ''
-
-    current = ""
-    previous = ""
+        return lines[-1]
 
     sentence = last_note['fields'][get_config().anki.sentence_field]['value']
-    if sentence:
-        found = False
-        for i, (line, clip_time) in enumerate(reversed(lines)):
-            similarity = similar(remove_html_tags(sentence), line)
-            logger.debug(f"Comparing: {remove_html_tags(sentence)} with {line} - Similarity: {similarity}")
-            if found:
-                previous = line
-                break
-            if similarity >= 0.60 or line in remove_html_tags(sentence):  # 80% similarity threshold
-                found = True
-                current = line
-
-    logger.debug(f"Current Line: {current}")
-    logger.debug(f"Previous Line: {previous}")
+    if not sentence:
+        return lines[-1]
 
-    if not current or not previous:
-        logger.debug("Couldn't find lines in history, using last two lines")
-        return lines[-1][0] if lines else '', lines[-2][0] if len(lines) > 1 else ''
+    for line in reversed(lines):
+        similarity = similar(remove_html_tags(sentence), line.text)
+        logger.debug(f"Comparing: {remove_html_tags(sentence)} with {line.text} - Similarity: {similarity}")
+        if similarity >= 0.60 or line.text in remove_html_tags(sentence):
+            return line
 
-    return current, previous
+    logger.debug("Couldn't find a match in history, using last event")
+    return lines[-1]
 
 
 def get_line_and_future_lines(last_note):
-    def similar(a, b):
-        return SequenceMatcher(None, a, b).ratio()
-    lines = list(line_history.items())
-
     if not last_note:
         return []
 
@@ -180,18 +248,27 @@ def get_line_and_future_lines(last_note):
     found_lines = []
     if sentence:
         found = False
-        for i, (line, clip_time) in enumerate(lines):
-            similarity = similar(remove_html_tags(sentence), line)
-            logger.debug(f"Comparing: {remove_html_tags(sentence)} with {line} - Similarity: {similarity}")
+        for line in line_history.values:
+            similarity = similar(remove_html_tags(sentence), line.text)
+            logger.debug(f"Comparing: {remove_html_tags(sentence)} with {line.text} - Similarity: {similarity}")
             if found:
-                found_lines.append(line)
-            if similarity >= 0.60 or line in remove_html_tags(sentence):  # 80% similarity threshold
+                found_lines.append(line.text)
+            if similarity >= 0.60 or line.text in remove_html_tags(sentence):  # 80% similarity threshold
                 found = True
-                found_lines.append(line)
+                found_lines.append(line.text)
     return found_lines
 
+def get_mined_line(last_note, lines):
+    if not last_note:
+        return lines[0]
+
+    sentence = last_note['fields'][get_config().anki.sentence_field]['value']
+    for line2 in lines:
+        similarity = similar(remove_html_tags(sentence), line2.text)
+        if similarity >= 0.60 or line2.text in remove_html_tags(sentence):
+            return line2
+    return lines[0]
+
 
 def get_time_of_line(line):
-    if line and line in line_history:
-        return line_history[line]
-    return initial_time
+    return line_history.get_time(line)