GameSentenceMiner 2.18.14__tar.gz → 2.18.16__tar.gz

{gamesentenceminer-2.18.14 → gamesentenceminer-2.18.16}/GameSentenceMiner/anki.py

@@ -233,12 +233,16 @@ def update_anki_card(last_note: 'AnkiCard', note=None, audio_path='', video_path
         config_app: 'ConfigApp' = gsm_state.config_app
         sentence = note['fields'].get(config.anki.sentence_field, last_note.get_field(config.anki.sentence_field))
         
-        use_voice, sentence, translation, new_ss_path = config_app.show_anki_confirmation_dialog(
+        use_voice, sentence, translation, new_ss_path, add_nsfw_tag = config_app.show_anki_confirmation_dialog(
             tango, sentence, assets.screenshot_path, assets.audio_path if update_audio_flag else None, translation, ss_time
         )
         note['fields'][config.anki.sentence_field] = sentence
         note['fields'][config.ai.anki_field] = translation
         assets.screenshot_path = new_ss_path or assets.screenshot_path
+        
+        # Add NSFW tag if checkbox was selected
+        if add_nsfw_tag:
+            tags.append("NSFW")
 
     # 5. If creating new media, store files in Anki's collection. Then update note fields.
     if not use_existing_files:
@@ -643,59 +647,10 @@ def start_monitoring_anki():
     obs_thread.start()
 
 
-# --- Anki Stats Kanji Extraction Utilities ---
+# --- Anki Stats Utilities ---
+# Note: Individual query functions have been removed in favor of the combined endpoint
+# All Anki statistics are now fetched through /api/anki_stats_combined
 
-def get_anki_earliest_date():
-    """
-    Fetches the earliest Anki card ID.
-    """
-    try:
-        note_ids = invoke("findCards", query="")
-        if not note_ids:
-            return 0
-        
-        # Return the first card ID as the "earliest"
-        # return note_ids[0]
-        return min(note_ids)
-        
-    except Exception as e:
-        logger.error(f"Failed to fetch kanji from Anki: {e}")
-        return 0
-
-def get_all_anki_first_field_kanji(start_timestamp = None, end_timestamp = None):
-    """
-    Fetch all notes from Anki and extract unique kanji from the first field of each note.
-    Returns a set of kanji characters.
-    Optional filtering by start_timestamp and end_timestamp on note IDs.
-    """
-    from GameSentenceMiner.web.stats import is_kanji
-    try:
-        note_ids = invoke("findNotes", query="")
-        if not note_ids:
-            return set()
-        
-        # Filter note IDs by start and end timestamps if provided
-        if (start_timestamp and end_timestamp):
-            note_ids = [nid for nid in note_ids if int(start_timestamp) <= nid <= int(end_timestamp)]
-            if not note_ids:
-                return set()
-        kanji_set = set()
-        batch_size = 1000
-        for i in range(0, len(note_ids), batch_size):
-            batch_ids = note_ids[i:i+batch_size]
-            notes_info = invoke("notesInfo", notes=batch_ids)
-            for note in notes_info:
-                fields = note.get("fields", {})
-                first_field = next(iter(fields.values()), None)
-                if first_field and "value" in first_field:
-                    first_field_value = first_field["value"]
-                    for char in first_field_value:
-                        if is_kanji(char):
-                            kanji_set.add(char)
-        return kanji_set
-    except Exception as e:
-        logger.error(f"Failed to fetch kanji from Anki: {e}")
-        return set()
 
 
 if __name__ == "__main__":

{gamesentenceminer-2.18.14 → gamesentenceminer-2.18.16}/GameSentenceMiner/obs.py

@@ -561,8 +561,7 @@ def get_replay_buffer_max_time_seconds():
                 if settings and 'max_time_sec' in settings:
                     return settings['max_time_sec']
                 else:
-                    logger.warning("Replay buffer settings received, but 'max_time_sec' key was not found.")
-                    return 0
+                    return 300
             else:
                 logger.warning(f"get_output_settings for replay_buffer failed: {response.status}")
                 return 0

{gamesentenceminer-2.18.14 → gamesentenceminer-2.18.16}/GameSentenceMiner/ui/anki_confirmation.py

@@ -36,6 +36,9 @@ class AnkiConfirmationDialog(tk.Toplevel):
         self.audio_path_label = None  # Store reference to audio path label
         self.tts_button = None  # Store reference to TTS button
         self.tts_status_label = None  # Store reference to TTS status label
+        
+        # NSFW tag option
+        self.nsfw_tag_var = tk.BooleanVar(value=False)
 
         self.title("Confirm Anki Card Details")
         self.result = None  # This will store the user's choice
@@ -152,6 +155,17 @@ class AnkiConfirmationDialog(tk.Toplevel):
                 
                 row += 1
 
+        # NSFW Tag Option
+        nsfw_frame = ttk.Frame(main_frame)
+        nsfw_frame.grid(row=row, column=0, columnspan=2, pady=10)
+        ttk.Checkbutton(
+            nsfw_frame,
+            text="Add NSFW tag?",
+            variable=self.nsfw_tag_var,
+            bootstyle="round-toggle"
+        ).pack(side="left", padx=5)
+        row += 1
+
         # Action Buttons
         button_frame = ttk.Frame(main_frame)
         button_frame.grid(row=row, column=0, columnspan=2, pady=15)
@@ -310,13 +324,13 @@ class AnkiConfirmationDialog(tk.Toplevel):
         # Clean up audio before closing
         self._cleanup_audio()
         # The screenshot_path is now correctly updated if the user chose a new one
-        self.result = (True, self.sentence_text.get("1.0", tk.END).strip(), self.translation_text.get("1.0", tk.END).strip() if self.translation_text else None, self.screenshot_path)
+        self.result = (True, self.sentence_text.get("1.0", tk.END).strip(), self.translation_text.get("1.0", tk.END).strip() if self.translation_text else None, self.screenshot_path, self.nsfw_tag_var.get())
         self.destroy()
 
     def _on_no_voice(self):
         # Clean up audio before closing
         self._cleanup_audio()
-        self.result = (False, self.sentence_text.get("1.0", tk.END).strip(), self.translation_text.get("1.0", tk.END).strip() if self.translation_text else None, self.screenshot_path)
+        self.result = (False, self.sentence_text.get("1.0", tk.END).strip(), self.translation_text.get("1.0", tk.END).strip() if self.translation_text else None, self.screenshot_path, self.nsfw_tag_var.get())
         self.destroy()
         
     def _on_cancel(self):

{gamesentenceminer-2.18.14 → gamesentenceminer-2.18.16}/GameSentenceMiner/util/db.py

@@ -451,9 +451,9 @@ class AIModelsTable(SQLiteDBTable):
 class GameLinesTable(SQLiteDBTable):
     _table = 'game_lines'
     _fields = ['game_name', 'line_text', 'screenshot_in_anki',
-               'audio_in_anki', 'screenshot_path', 'audio_path', 'replay_path', 'translation', 'timestamp', 'original_game_name']
+               'audio_in_anki', 'screenshot_path', 'audio_path', 'replay_path', 'translation', 'timestamp', 'original_game_name', 'game_id']
     _types = [str,  # Includes primary key type
-              str, str, str, str, str, str, str, str, float, str]
+              str, str, str, str, str, str, str, str, float, str, str]
     _pk = 'id'
     _auto_increment = False  # Use string IDs
 
@@ -468,7 +468,8 @@ class GameLinesTable(SQLiteDBTable):
                  audio_path: Optional[str] = None,
                  replay_path: Optional[str] = None,
                  translation: Optional[str] = None,
-                 original_game_name: Optional[str] = None):
+                 original_game_name: Optional[str] = None,
+                 game_id: Optional[str] = None):
         self.id = id
         self.game_name = game_name
         self.line_text = line_text
@@ -481,6 +482,7 @@ class GameLinesTable(SQLiteDBTable):
         self.replay_path = replay_path if replay_path is not None else ''
         self.translation = translation if translation is not None else ''
         self.original_game_name = original_game_name if original_game_name is not None else ''
+        self.game_id = game_id if game_id is not None else ''
 
     @classmethod
     def get_all_lines_for_scene(cls, game_name: str) -> List['GameLinesTable']:
@@ -605,7 +607,7 @@ class StatsRollupTable(SQLiteDBTable):
         stats.anki_cards_created += anki_cards_created
         stats.time_spent_mining += time_spent_mining
         stats.save()
-        
+
 # Ensure database directory exists and return path
 def get_db_directory(test=False, delete_test=False) -> str:
     if platform == 'win32':  # Windows
@@ -659,7 +661,10 @@ if os.path.exists(db_path):
 
 gsm_db = SQLiteDB(db_path)
 
-for cls in [AIModelsTable, GameLinesTable]:
+# Import GamesTable after gsm_db is created to avoid circular import
+from GameSentenceMiner.util.games_table import GamesTable
+
+for cls in [AIModelsTable, GameLinesTable, GamesTable]:
     cls.set_db(gsm_db)
     # Uncomment to start fresh every time
     # cls.drop()
@@ -680,8 +685,7 @@ def check_and_run_migrations():
             # Copy and cast data from old column to new column
             GameLinesTable.alter_column_type('timestamp_old', 'timestamp', 'REAL')
             logger.info("Migrated 'timestamp' column to REAL type in GameLinesTable.")
-            
-            
+    
     migrate_timestamp()
         
 check_and_run_migrations()

gamesentenceminer-2.18.16/GameSentenceMiner/util/games_table.py

@@ -0,0 +1,320 @@
+import uuid
+from typing import Optional, List, Dict
+
+from GameSentenceMiner.util.db import SQLiteDBTable
+from GameSentenceMiner.util.configuration import logger
+
+
+class GamesTable(SQLiteDBTable):
+    _table = 'games'
+    _fields = [
+        'deck_id', 'title_original', 'title_romaji', 'title_english',
+        'type',
+        'description', 'image', 'character_count', 'difficulty', 'links', 'completed',
+        'manual_overrides'
+    ]
+    _types = [
+        str,      # id (primary key)
+        int,      # deck_id
+        str,      # title_original
+        str,      # title_romaji
+        str,      # title_english
+        str,      # type (string)
+        str,      # description
+        str,      # image (base64)
+        int,      # character_count
+        int,      # difficulty
+        list,     # links (stored as JSON)
+        bool,     # completed
+        list      # manual_overrides (stored as JSON)
+    ]
+    _pk = 'id'
+    _auto_increment = False  # UUID-based primary key
+
+    def __init__(
+        self,
+        id: Optional[str] = None,
+        deck_id: Optional[int] = None,
+        title_original: Optional[str] = None,
+        title_romaji: Optional[str] = None,
+        title_english: Optional[str] = None,
+        type: Optional[str] = None,
+        description: Optional[str] = None,
+        image: Optional[str] = None,
+        character_count: int = 0,
+        difficulty: Optional[int] = None,
+        links: Optional[List[Dict]] = None,
+        completed: bool = False,
+        manual_overrides: Optional[List[str]] = None
+    ):
+        self.id = id if id else str(uuid.uuid4())
+        self.deck_id = deck_id
+        self.title_original = title_original if title_original else ''
+        self.title_romaji = title_romaji if title_romaji else ''
+        self.title_english = title_english if title_english else ''
+        self.type = type if type else ''
+        self.description = description if description else ''
+        self.image = image if image else ''
+        self.character_count = character_count
+        self.difficulty = difficulty
+        self.links = links if links else []
+        self.completed = completed
+        self.manual_overrides = manual_overrides if manual_overrides else []
+
+    @classmethod
+    def get_by_deck_id(cls, deck_id: int) -> Optional['GamesTable']:
+        """Get a game by its jiten.moe deck ID."""
+        row = cls._db.fetchone(
+            f"SELECT * FROM {cls._table} WHERE deck_id=?", (deck_id,))
+        return cls.from_row(row) if row else None
+
+    @classmethod
+    def get_by_title(cls, title_original: str) -> Optional['GamesTable']:
+        """Get a game by its original title."""
+        row = cls._db.fetchone(
+            f"SELECT * FROM {cls._table} WHERE title_original=?", (title_original,))
+        return cls.from_row(row) if row else None
+
+    @classmethod
+    def get_or_create_by_name(cls, game_name: str) -> 'GamesTable':
+        """
+        Get an existing game by name, or create a new one if it doesn't exist.
+        This is the primary method for automatically linking game_lines to games.
+        
+        Args:
+            game_name: The original game name (from game_lines.game_name)
+            
+        Returns:
+            GamesTable: The existing or newly created game record
+        """
+        # Try to find existing game
+        existing = cls.get_by_title(game_name)
+        if existing:
+            return existing
+        
+        # Create new game with minimal info
+        new_game = cls(
+            title_original=game_name,
+            title_romaji='',
+            title_english='',
+            description='',
+            difficulty=None,
+            completed=False
+        )
+        new_game.save()
+        logger.info(f"Auto-created new game record: {game_name} (id={new_game.id})")
+        return new_game
+
+    @classmethod
+    def get_all_completed(cls) -> List['GamesTable']:
+        """Get all completed games."""
+        rows = cls._db.fetchall(
+            f"SELECT * FROM {cls._table} WHERE completed=1")
+        return [cls.from_row(row) for row in rows]
+
+    @classmethod
+    def get_all_in_progress(cls) -> List['GamesTable']:
+        """Get all games that are in progress (not completed)."""
+        rows = cls._db.fetchall(
+            f"SELECT * FROM {cls._table} WHERE completed=0")
+        return [cls.from_row(row) for row in rows]
+
+    @classmethod
+    def get_start_date(cls, game_id: str) -> Optional[float]:
+        """
+        Get the start date (timestamp of first line) for a game.
+        Returns Unix timestamp (float) or None if no lines exist.
+        """
+        from GameSentenceMiner.util.db import GameLinesTable
+        result = GameLinesTable._db.fetchone(
+            f"SELECT MIN(timestamp) FROM {GameLinesTable._table} WHERE game_id=?",
+            (game_id,)
+        )
+        return result[0] if result and result[0] else None
+
+    @classmethod
+    def get_last_played_date(cls, game_id: str) -> Optional[float]:
+        """
+        Get the last played date (timestamp of most recent line) for a game.
+        Returns Unix timestamp (float) or None if no lines exist.
+        """
+        from GameSentenceMiner.util.db import GameLinesTable
+        result = GameLinesTable._db.fetchone(
+            f"SELECT MAX(timestamp) FROM {GameLinesTable._table} WHERE game_id=?",
+            (game_id,)
+        )
+        return result[0] if result and result[0] else None
+
+    def is_field_manual(self, field_name: str) -> bool:
+        """
+        Check if a field has been manually edited and should not be auto-updated.
+        
+        Args:
+            field_name: The name of the field to check
+            
+        Returns:
+            True if the field is manually overridden, False otherwise
+        """
+        return field_name in self.manual_overrides
+    
+    def mark_field_manual(self, field_name: str):
+        """
+        Mark a field as manually edited so it won't be auto-updated.
+        
+        Args:
+            field_name: The name of the field to mark as manual
+        """
+        if field_name not in self.manual_overrides and field_name in self._fields:
+            self.manual_overrides.append(field_name)
+            logger.debug(f"Marked field '{field_name}' as manually overridden for game {self.id}")
+
+    def update_all_fields_manual(
+        self,
+        deck_id: Optional[int] = None,
+        title_original: Optional[str] = None,
+        title_romaji: Optional[str] = None,
+        title_english: Optional[str] = None,
+        type: Optional[str] = None,
+        description: Optional[str] = None,
+        image: Optional[str] = None,
+        character_count: Optional[int] = None,
+        difficulty: Optional[int] = None,
+        links: Optional[List[Dict]] = None,
+        completed: Optional[bool] = None
+    ):
+        """
+        Update all fields of the game at once. Only provided fields will be updated.
+        Fields that are updated will be automatically marked as manually overridden.
+        
+        Args:
+            deck_id: jiten.moe deck ID
+            title_original: Original Japanese title
+            title_romaji: Romanized title
+            title_english: English translated title
+            description: Game description
+            image: Base64-encoded image data
+            character_count: Total character count
+            difficulty: Difficulty rating
+            links: List of link objects
+            completed: Whether the game is completed
+        """
+        if deck_id is not None:
+            self.deck_id = deck_id
+            self.mark_field_manual('deck_id')
+        if title_original is not None:
+            self.title_original = title_original
+            self.mark_field_manual('title_original')
+        if title_romaji is not None:
+            self.title_romaji = title_romaji
+            self.mark_field_manual('title_romaji')
+        if title_english is not None:
+            self.title_english = title_english
+            self.mark_field_manual('title_english')
+        if type is not None:
+            self.type = type
+            self.mark_field_manual('type')
+        if description is not None:
+            self.description = description
+            self.mark_field_manual('description')
+        if image is not None:
+            self.image = image
+            self.mark_field_manual('image')
+        if character_count is not None:
+            self.character_count = character_count
+            self.mark_field_manual('character_count')
+        if difficulty is not None:
+            self.difficulty = difficulty
+            self.mark_field_manual('difficulty')
+        if links is not None:
+            self.links = links
+            self.mark_field_manual('links')
+        if completed is not None:
+            self.completed = completed
+            self.mark_field_manual('completed')
+        
+        self.save()
+        logger.info(f"Updated game {self.id} ({self.title_original})")
+
+    def update_all_fields_from_jiten(
+        self,
+        deck_id: Optional[int] = None,
+        title_original: Optional[str] = None,
+        title_romaji: Optional[str] = None,
+        title_english: Optional[str] = None,
+        type: Optional[str] = None,
+        description: Optional[str] = None,
+        image: Optional[str] = None,
+        character_count: Optional[int] = None,
+        difficulty: Optional[int] = None,
+        links: Optional[List[Dict]] = None,
+        completed: Optional[bool] = None
+    ):
+        """
+        Update all fields of the game at once. Only provided fields will be updated.
+        
+        Args:
+            deck_id: jiten.moe deck ID
+            title_original: Original Japanese title
+            title_romaji: Romanized title
+            title_english: English translated title
+            description: Game description
+            image: Base64-encoded image data
+            character_count: Total character count
+            difficulty: Difficulty rating
+            links: List of link objects
+            completed: Whether the game is completed
+        """
+        if deck_id is not None:
+            self.deck_id = deck_id
+        if title_original is not None:
+            self.title_original = title_original
+        if title_romaji is not None:
+            self.title_romaji = title_romaji
+        if title_english is not None:
+            self.title_english = title_english
+        if type is not None:
+            self.type = type
+        if description is not None:
+            self.description = description
+        if image is not None:
+            self.image = image
+        if character_count is not None:
+            self.character_count = character_count
+        if difficulty is not None:
+            self.difficulty = difficulty
+        if links is not None:
+            self.links = links
+        if completed is not None:
+            self.completed = completed        
+        self.save()
+        logger.info(f"Updated game {self.id} ({self.title_original})")
+
+    def add_link(self, link_type: int, url: str, link_id: Optional[int] = None):
+        """
+        Add a link to the game's links array and persist to database.
+        
+        Args:
+            link_type: Type of link (e.g., 4 for AniList, 5 for MyAnimeList)
+            url: URL of the link
+            link_id: Optional link ID
+            
+        Note:
+            Changes are automatically saved to the database.
+        """
+        new_link = {
+            'linkType': link_type,
+            'url': url,
+            'deckId': self.deck_id
+        }
+        if link_id is not None:
+            new_link['linkId'] = link_id
+        
+        self.links.append(new_link)
+        self.save()
+
+    def get_lines(self) -> List:
+        """Get all lines associated with this game."""
+        from GameSentenceMiner.util.db import GameLinesTable
+        rows = GameLinesTable._db.fetchall(
+            f"SELECT * FROM {GameLinesTable._table} WHERE game_id=?", (self.id,))
+        return [GameLinesTable.from_row(row) for row in rows]

{gamesentenceminer-2.18.14 → gamesentenceminer-2.18.16}/GameSentenceMiner/vad.py

@@ -195,11 +195,11 @@ class WhisperVADProcessor(VADProcessor):
         # If both mined text and Whisper transcription are available, compare their similarity
         if text_mined and text:
             from rapidfuzz import fuzz
-            similarity = fuzz.partial_ratio(text_mined, text)
-            logger.info(f"Whisper transcription: '{text}' | Mined text: '{text_mined}' | Partial similarity: {similarity:.1f}")
+            similarity = fuzz.ratio(text_mined, text)
+            logger.info(f"Whisper transcription: '{text}' | Mined text: '{text_mined}' | Full similarity: {similarity:.1f}")
             text_similarity = similarity
             if similarity < 20:
-                logger.info(f"Partial similarity {similarity:.1f} is below threshold, skipping voice activity.")
+                logger.info(f"Full similarity {similarity:.1f} is below threshold, skipping voice activity.")
                 return []
 
         # Process the segments to extract tokens, timestamps, and confidence