GameSentenceMiner 2.16.6__py3-none-any.whl → 2.16.8__py3-none-any.whl

GameSentenceMiner/config_gui.py

@@ -366,6 +366,7 @@ class ConfigApp:
         self.language_value = tk.StringVar(value=self.settings.vad.language)
         self.cut_and_splice_segments_value = tk.BooleanVar(value=self.settings.vad.cut_and_splice_segments)
         self.splice_padding_value = tk.StringVar(value=str(self.settings.vad.splice_padding) if self.settings.vad.splice_padding else "")
+        self.use_vad_filter_for_whisper_value = tk.BooleanVar(value=self.settings.vad.use_vad_filter_for_whisper)
         
         # Advanced Settings
         self.audio_player_path_value = tk.StringVar(value=self.settings.advanced.audio_player_path)
@@ -598,6 +599,7 @@ class ConfigApp:
                 cut_and_splice_segments=self.cut_and_splice_segments_value.get(),
                 splice_padding=float(self.splice_padding_value.get()) if self.splice_padding_value.get() else 0.0,
                 use_cpu_for_inference=self.use_cpu_for_inference_value.get(),
+                use_vad_filter_for_whisper=self.use_vad_filter_for_whisper_value.get(),
             ),
             advanced=Advanced(
                 audio_player_path=self.audio_player_path_value.get(),
@@ -1150,6 +1152,12 @@ class ConfigApp:
         ttk.Checkbutton(vad_frame, variable=self.use_cpu_for_inference_value, bootstyle="round-toggle").grid(row=self.current_row, column=1, sticky='W', pady=2)
         self.current_row += 1
         
+        # TODO Add Localization
+        use_vad_filter_for_whisper_i18n = vad_i18n.get('use_vad_filter_for_whisper', {})
+        HoverInfoLabelWidget(vad_frame, text=use_vad_filter_for_whisper_i18n.get('label', 'Use VAD Filter for Whisper'), tooltip=use_vad_filter_for_whisper_i18n.get('tooltip', 'Uses Silero to Filter out Non-Voiced Segments before Transcribing with Whisper.'), row=self.current_row, column=0)
+        ttk.Checkbutton(vad_frame, variable=self.use_vad_filter_for_whisper_value, bootstyle="round-toggle").grid(row=self.current_row, column=1, sticky='W', pady=2)
+        self.current_row += 1
+        
         # Add Reset Button
         self.add_reset_button(vad_frame, "vad", self.current_row, column=0, recreate_tab=self.create_vad_tab)
 

GameSentenceMiner/util/configuration.py

@@ -561,6 +561,7 @@ class VAD:
     cut_and_splice_segments: bool = False
     splice_padding: float = 0.1
     use_cpu_for_inference: bool = False
+    use_vad_filter_for_whisper: bool = True
 
     def is_silero(self):
         return self.selected_vad_model == SILERO or self.backup_vad_model == SILERO

GameSentenceMiner/util/db.py

@@ -1,16 +1,22 @@
 
 
+from datetime import datetime
 import json
 import os
+import random
 import shutil
 import sqlite3
 from sys import platform
 import time
 from typing import Any, Dict, List, Optional, Tuple, Union, Type, TypeVar
 import threading
+import uuid
+
+import pytz
+from datetime import timedelta
 
 from GameSentenceMiner.util.text_log import GameLine
-from GameSentenceMiner.util.configuration import logger, is_dev
+from GameSentenceMiner.util.configuration import get_stats_config, logger, is_dev
 import gzip
 
 
@@ -23,6 +29,14 @@ class SQLiteDB:
     def __init__(self, db_path: str):
         self.db_path = db_path
         self._lock = threading.Lock()
+        
+    def backup(self, backup_path: str):
+        """ Create a backup of the database using built in SQLite backup API. """
+        with self._lock, sqlite3.connect(self.db_path, check_same_thread=False) as conn:
+            with sqlite3.connect(backup_path, check_same_thread=False) as backup_conn:
+                conn.backup(backup_conn)
+                if is_dev:
+                    logger.debug(f"Database backed up to {backup_path}")
 
     def execute(self, query: str, params: Union[Tuple, Dict] = (), commit: bool = False) -> sqlite3.Cursor:
         with self._lock, sqlite3.connect(self.db_path, check_same_thread=False) as conn:
@@ -341,7 +355,7 @@ class GameLinesTable(SQLiteDBTable):
         self.game_name = game_name
         self.line_text = line_text
         self.context = context
-        self.timestamp = timestamp if timestamp is not None else time.time()
+        self.timestamp = timestamp if timestamp is not None else datetime.now().timestamp()
         self.screenshot_in_anki = screenshot_in_anki if screenshot_in_anki is not None else ''
         self.audio_in_anki = audio_in_anki if audio_in_anki is not None else ''
         self.screenshot_path = screenshot_path if screenshot_path is not None else ''
@@ -388,6 +402,20 @@ class GameLinesTable(SQLiteDBTable):
         # logger.info("Adding GameLine to DB: %s", new_line)
         new_line.add()
         return new_line
+    
+    @classmethod
+    def add_lines(cls, gamelines: List[GameLine]):
+        new_lines = [cls(id=gl.id, game_name=gl.scene,
+                         line_text=gl.text, timestamp=gl.time.timestamp()) for gl in gamelines]
+        # logger.info("Adding %d GameLines to DB", len(new_lines))
+        params = [(line.id, line.game_name, line.line_text, line.timestamp, line.screenshot_in_anki,
+                   line.audio_in_anki, line.screenshot_path, line.audio_path, line.replay_path, line.translation)
+                  for line in new_lines]
+        cls._db.executemany(
+            f"INSERT INTO {cls._table} (id, game_name, line_text, timestamp, screenshot_in_anki, audio_in_anki, screenshot_path, audio_path, replay_path, translation) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
+            params,
+            commit=True
+        )
 
 
 def get_db_directory():
@@ -441,3 +469,56 @@ for cls in [AIModelsTable, GameLinesTable]:
     # Uncomment to start fresh every time
     # cls.drop()
     # cls.set_db(gsm_db)  # --- IGNORE ---
+    
+# import random
+# import uuid
+# from datetime import datetime
+# from GameSentenceMiner.util.text_log import GameLine
+# from GameSentenceMiner.util.db import GameLinesTable
+
+# List of common Japanese characters (kanji, hiragana, katakana)
+# japanese_chars = (
+#     "あいうえおかきくけこさしすせそたちつてとなにぬねのはひふへほまみむめもやゆよらりるれろわをん"
+#     "アイウエオカキクケコサシスセソタチツテトナニヌネノハヒフヘホマミムメモヤユヨラリルレロワヲン"
+#     "亜唖娃阿哀愛挨悪握圧扱宛嵐安暗案闇以衣位囲医依委威為胃尉異移維緯"
+#     # ... (add more kanji for more variety)
+# )
+
+# def random_japanese_text(length):
+#     return ''.join(random.choices(japanese_chars, k=length))
+
+# batch_size = 1000
+# lines_batch = []
+
+# def random_datetime(start_year=2024, end_year=2026):
+#     start = datetime(start_year, 1, 1, 0, 0, 0)
+#     end = datetime(end_year, 12, 31, 23, 59, 59)
+#     delta = end - start
+#     random_seconds = random.randint(0, int(delta.total_seconds()))
+#     return start + timedelta(seconds=random_seconds)
+
+# from datetime import timedelta
+
+# for i in range(500000):  # Adjust for desired number of lines
+#     line_text = random_japanese_text(random.randint(25, 40))
+#     lines_batch.append(GameLine(
+#         id=str(uuid.uuid1()),
+#         text=line_text,
+#         time=random_datetime(),
+#         prev=None,
+#         next=None,
+#         index=i,
+#         scene="RandomScene"
+#     ))
+    
+#     if len(lines_batch) >= batch_size:
+#         GameLinesTable.add_lines(lines_batch)
+#         lines_batch = []
+#     if i % 1000 == 0:
+#         print(f"Inserted {i} lines...")
+
+# # Insert any remaining lines
+# if lines_batch:
+#     GameLinesTable.add_lines(lines_batch)
+
+print("Done populating GameLinesDB with random Japanese text.")

GameSentenceMiner/util/ffmpeg.py

@@ -495,6 +495,7 @@ def get_video_timings(video_path, game_line, anki_card_creation_time=None):
     total_seconds = file_length - time_delta.total_seconds()
     total_seconds_after_offset = total_seconds + get_config().audio.beginning_offset
     if total_seconds < 0 or total_seconds >= file_length:
+        logger.debug(f"get_video_timings: file_mod_time={file_mod_time}, file_length={file_length}, time_delta={time_delta}, total_seconds={total_seconds}, total_seconds_after_offset={total_seconds_after_offset}")
         logger.error("Line mined is outside of the replay buffer! Defaulting to the last 30 seconds of the replay buffer.")
         logger.info("Recommend either increasing replay buffer length in OBS Settings or mining faster.")
         return max(file_length - 30, 0), 0, max(file_length - 30, 0), file_length

GameSentenceMiner/vad.py

@@ -185,7 +185,7 @@ class WhisperVADProcessor(VADProcessor):
         # Transcribe the audio using Whisper
         with warnings.catch_warnings():
             warnings.simplefilter("ignore")
-            result: WhisperResult = self.vad_model.transcribe(temp_wav, vad=True, language=get_config().vad.language,
+            result: WhisperResult = self.vad_model.transcribe(temp_wav, vad=True, language=get_config().vad.language, vad_filter=get_config().vad.use_vad_filter_for_whisper,
                                                              temperature=0.0)
         voice_activity = []
 
@@ -406,6 +406,7 @@ def test_vad_processors():
     get_config().vad.cut_and_splice_segments = False
     get_config().vad.trim_beginning = True
     get_config().vad.add_audio_on_no_results = True
+    get_config().vad.use_vad_filter_for_whisper = False
     for processor, out_name in processors:
         logger.info("Testing Trim Audio with " + processor.vad_system_name)
         out_path = os.path.join(output_dir, out_name.replace("after_splice_", "after_trim_"))

GameSentenceMiner/web/database_api.py

@@ -12,6 +12,7 @@ import regex
 
 from GameSentenceMiner.util.db import GameLinesTable
 from GameSentenceMiner.util.configuration import get_stats_config, logger, get_config, save_current_config, save_stats_config
+from GameSentenceMiner.util.text_log import GameLine
 from GameSentenceMiner.web.stats import (
     calculate_kanji_frequency, calculate_heatmap_data, calculate_total_chars_per_game,
     calculate_reading_time_per_game, calculate_reading_speed_per_game,
@@ -304,7 +305,7 @@ def register_database_api_routes(app):
                 'streak_requirement_hours': config.streak_requirement_hours,
                 'reading_hours_target': config.reading_hours_target,
                 'character_count_target': config.character_count_target,
-                'games_target': config.games_target
+                'games_target': config.games_target,
             }), 200
         except Exception as e:
             logger.error(f"Error getting settings: {e}")
@@ -806,6 +807,8 @@ def register_database_api_routes(app):
         daily_data = defaultdict(lambda: defaultdict(lambda: {'lines': 0, 'chars': 0}))
         
         
+        
+        
         # start_time = time.perf_counter()
         # for line in all_lines:
         #     day_str = datetime.date.fromtimestamp(float(line.timestamp)).strftime('%Y-%m-%d')
@@ -975,24 +978,31 @@ def register_database_api_routes(app):
                 csv_reader = csv.DictReader(file_io, quoting=csv.QUOTE_MINIMAL, skipinitialspace=True)
                 
                 # Process CSV rows
-                imported_lines = []
                 games_set = set()
                 errors = []
-                seen_uuids = set()  # Track UUIDs within this import batch
                 
+                all_lines = GameLinesTable.all()
+                existing_uuids = {line.id for line in all_lines}
+                batch_size = 1000  # For logging progress
+                batch_insert = []
+                imported_count = 0
+                
+                def get_line_hash(uuid: str, line_text: str) -> str:
+                    return uuid + '|' + line_text.strip()
+
                 for row_num, row in enumerate(csv_reader):
                     try:
                         # Extract and validate required fields
-                        uuid = row.get('uuid', '').strip()
-                        name = row.get('name', '').strip()
+                        game_uuid = row.get('uuid', '').strip()
+                        game_name = row.get('name', '').strip()
                         line = row.get('line', '').strip()
                         time_str = row.get('time', '').strip()
                         
                         # Validate required fields
-                        if not uuid:
+                        if not game_uuid:
                             errors.append(f"Row {row_num}: Missing UUID")
                             continue
-                        if not name:
+                        if not game_name:
                             errors.append(f"Row {row_num}: Missing name")
                             continue
                         if not line:
@@ -1002,12 +1012,13 @@ def register_database_api_routes(app):
                             errors.append(f"Row {row_num}: Missing time")
                             continue
                         
-                        # Check for duplicates within this import batch
-                        if uuid in seen_uuids:
-                            logger.info(f"Skipping duplicate UUID within import batch: {uuid}")
-                            continue
-                        seen_uuids.add(uuid)
+                        line_hash = get_line_hash(game_uuid, line)
                         
+                        # Check if this line already exists in database
+                        if line_hash in existing_uuids:
+                            logger.info(f"Skipping duplicate UUID already in database: {line_hash}")
+                            continue
+
                         # Convert time to timestamp
                         try:
                             timestamp = float(time_str)
@@ -1018,37 +1029,52 @@ def register_database_api_routes(app):
                         # Clean up line text (remove extra whitespace and newlines)
                         line_text = line.strip()
                         
-                        # Check if this UUID already exists in database
-                        existing_line = GameLinesTable.get(uuid)
-                        if existing_line:
-                            logger.info(f"Skipping duplicate UUID already in database: {uuid}")
-                            continue
-                        
                         # Create GameLinesTable entry
-                        game_line = GameLinesTable(
-                            id=uuid,
-                            game_name=name,
-                            line_text=line_text,
-                            timestamp=timestamp
-                        )
+                        # Convert timestamp float to datetime object
+                        dt = datetime.datetime.fromtimestamp(timestamp)
+                        batch_insert.append(GameLine(
+                            id=line_hash,
+                            text=line_text,
+                            scene=game_name,
+                            time=dt,
+                            prev=None,
+                            next=None,
+                            index=0,
+                        ))
+                        
+                        logger.info(f"Batch insert size: {len(batch_insert)}")
+                        
+                        existing_uuids.add(line_hash)  # Add to existing to prevent duplicates in same import
                         
-                        imported_lines.append(game_line)
-                        games_set.add(name)
+                        if len(batch_insert) >= batch_size:
+                            logger.info(f"Importing batch of {len(batch_insert)} lines...")
+                            GameLinesTable.add_lines(batch_insert)
+                            imported_count += len(batch_insert)
+                            batch_insert = []
+                        games_set.add(game_name)
                         
                     except Exception as e:
+                        logger.error(f"Error processing row {row_num}: {e}")
                         errors.append(f"Row {row_num}: Error processing row - {str(e)}")
                         continue
+                    
+                # Insert the rest of the batch
+                if batch_insert:
+                    logger.info(f"Importing final batch of {len(batch_insert)} lines...")
+                    GameLinesTable.add_lines(batch_insert)
+                    imported_count += len(batch_insert)
+                    batch_insert = []
                 
-                # Import lines into database
-                imported_count = 0
-                for game_line in imported_lines:
-                    try:
-                        game_line.add()
-                        imported_count += 1
-                    except Exception as e:
-                        logger.error(f"Failed to import line {game_line.id}: {e}")
-                        errors.append(f"Failed to import line {game_line.id}: {str(e)}")
-                
+                # # Import lines into database
+                # imported_count = 0
+                # for game_line in imported_lines:
+                #     try:
+                #         game_line.add()
+                #         imported_count += 1
+                #     except Exception as e:
+                #         logger.error(f"Failed to import line {game_line.id}: {e}")
+                #         errors.append(f"Failed to import line {game_line.id}: {str(e)}")
+
                 # Prepare response
                 response_data = {
                     'message': f'Successfully imported {imported_count} lines from {len(games_set)} games',
@@ -1063,6 +1089,8 @@ def register_database_api_routes(app):
                 
                 logger.info(f"ExStatic import completed: {imported_count} lines from {len(games_set)} games")
                 
+                logger.info(f"Import response: {response_data}")
+                
                 return jsonify(response_data), 200
                 
             except csv.Error as e:

GameSentenceMiner/web/static/js/search.js

@@ -186,7 +186,7 @@ class SentenceSearchApp {
         
         // Format timestamp to ISO format
         const date = new Date(result.timestamp * 1000);
-        const formattedDate = date.toISOString().split('T')[0] + ' ' + date.toTimeString().split(' ')[0];
+        const formattedDate = `${date.getFullYear()}-${String(date.getMonth() + 1).padStart(2, '0')}-${String(date.getDate()).padStart(2, '0')} ${date.toTimeString().split(' ')[0]}`;
         
         div.innerHTML = `
             <div class="result-sentence">${highlightedText}</div>

GameSentenceMiner/web/static/js/stats.js

@@ -128,7 +128,7 @@ document.addEventListener('DOMContentLoaded', function () {
             for (let day = 0; day < 7; day++) {
                 const date = grid[day][week];
                 if (date) {
-                    const dateStr = date.toISOString().split('T')[0];
+                    const dateStr = `${date.getFullYear()}-${String(date.getMonth() + 1).padStart(2, '0')}-${String(date.getDate()).padStart(2, '0')}`;
                     const activity = yearData[dateStr] || 0;
                     dates.push({ date: dateStr, activity: activity });
                 }
@@ -154,7 +154,8 @@ document.addEventListener('DOMContentLoaded', function () {
         }
         
         // Calculate current streak from today backwards
-        const today = new Date().toISOString().split('T')[0];
+        const date = new Date();
+        const today = `${date.getFullYear()}-${String(date.getMonth() + 1).padStart(2, '0')}-${String(date.getDate()).padStart(2, '0')}`;
         
         // Find today's index or the most recent date before today
         let todayIndex = -1;
@@ -182,7 +183,10 @@ document.addEventListener('DOMContentLoaded', function () {
             // Group timestamps by day for this year
             const dailyTimestamps = {};
             for (const line of allLinesForYear) {
-                const dateStr = new Date(parseFloat(line.timestamp) * 1000).toISOString().split('T')[0];
+                const ts = parseFloat(line.timestamp);
+                if (isNaN(ts)) continue;
+                const dateObj = new Date(ts * 1000);
+                const dateStr = `${dateObj.getFullYear()}-${String(dateObj.getMonth() + 1).padStart(2, '0')}-${String(dateObj.getDate()).padStart(2, '0')}`;
                 if (!dailyTimestamps[dateStr]) {
                     dailyTimestamps[dateStr] = [];
                 }
@@ -326,7 +330,7 @@ document.addEventListener('DOMContentLoaded', function () {
                     
                     const date = grid[day][week];
                     if (date) {
-                        const dateStr = date.toISOString().split('T')[0];
+                        const dateStr = `${date.getFullYear()}-${String(date.getMonth() + 1).padStart(2, '0')}-${String(date.getDate()).padStart(2, '0')}`;
                         const activity = yearData[dateStr] || 0;
                         
                         if (activity > 0 && maxActivity > 0) {
@@ -881,11 +885,14 @@ document.addEventListener('DOMContentLoaded', function () {
             // Group by day and calculate reading time using AFK timer logic
             const dailyTimestamps = {};
             for (const line of recentData) {
-                const dateStr = new Date(line.timestamp * 1000).toISOString().split('T')[0];
+                const ts = parseFloat(line.timestamp);
+                if (isNaN(ts)) continue;
+                const dateObj = new Date(ts * 1000);
+                const dateStr = `${dateObj.getFullYear()}-${String(dateObj.getMonth() + 1).padStart(2, '0')}-${String(dateObj.getDate()).padStart(2, '0')}`;
                 if (!dailyTimestamps[dateStr]) {
                     dailyTimestamps[dateStr] = [];
                 }
-                dailyTimestamps[dateStr].push(line.timestamp);
+                dailyTimestamps[dateStr].push(ts);
             }
             
             for (const [dateStr, timestamps] of Object.entries(dailyTimestamps)) {
@@ -906,14 +913,20 @@ document.addEventListener('DOMContentLoaded', function () {
         } else if (metricType === 'characters') {
             // Group by day and sum characters
             for (const line of recentData) {
-                const dateStr = new Date(line.timestamp * 1000).toISOString().split('T')[0];
+                const ts = parseFloat(line.timestamp);
+                if (isNaN(ts)) continue;
+                const dateObj = new Date(ts * 1000);
+                const dateStr = `${dateObj.getFullYear()}-${String(dateObj.getMonth() + 1).padStart(2, '0')}-${String(dateObj.getDate()).padStart(2, '0')}`;
                 dailyTotals[dateStr] = (dailyTotals[dateStr] || 0) + (line.characters || 0);
             }
         } else if (metricType === 'games') {
             // Group by day and count unique games
             const dailyGames = {};
             for (const line of recentData) {
-                const dateStr = new Date(line.timestamp * 1000).toISOString().split('T')[0];
+                const ts = parseFloat(line.timestamp);
+                if (isNaN(ts)) continue;
+                const dateObj = new Date(ts * 1000);
+                const dateStr = `${dateObj.getFullYear()}-${String(dateObj.getMonth() + 1).padStart(2, '0')}-${String(dateObj.getDate()).padStart(2, '0')}`;
                 if (!dailyGames[dateStr]) {
                     dailyGames[dateStr] = new Set();
                 }
@@ -1141,14 +1154,19 @@ document.addEventListener('DOMContentLoaded', function () {
     function loadDashboardData(data = null) {
         function updateTodayOverview(allLinesData) {
             // Get today's date string (YYYY-MM-DD)
+            // Get today's date string (YYYY-MM-DD), timezone aware (local time)
             const today = new Date();
-            const todayStr = today.toISOString().split('T')[0];
+            const pad = n => n.toString().padStart(2, '0');
+            const todayStr = `${today.getFullYear()}-${pad(today.getMonth() + 1)}-${pad(today.getDate())}`;
             document.getElementById('todayDate').textContent = todayStr;
 
             // Filter lines for today
             const todayLines = (allLinesData || []).filter(line => {
                 if (!line.timestamp) return false;
-                const lineDate = new Date(parseFloat(line.timestamp) * 1000).toISOString().split('T')[0];
+                const ts = parseFloat(line.timestamp);
+                if (isNaN(ts)) return false;
+                const dateObj = new Date(ts * 1000);
+                const lineDate = `${dateObj.getFullYear()}-${pad(dateObj.getMonth() + 1)}-${pad(dateObj.getDate())}`;
                 return lineDate === todayStr;
             });
 
@@ -1195,7 +1213,13 @@ document.addEventListener('DOMContentLoaded', function () {
                 .map(l => parseFloat(l.timestamp))
                 .filter(ts => !isNaN(ts))
                 .sort((a, b) => a - b);
-            const afkTimerSeconds = 120;
+            // Get AFK timer from settings modal if available
+            let afkTimerSeconds = 120; // default
+            const afkTimerInput = document.getElementById('afkTimer');
+            if (afkTimerInput && afkTimerInput.value) {
+                const parsed = parseInt(afkTimerInput.value, 10);
+                if (!isNaN(parsed) && parsed > 0) afkTimerSeconds = parsed;
+            }
             if (timestamps.length >= 2) {
                 for (let i = 1; i < timestamps.length; i++) {
                     const gap = timestamps[i] - timestamps[i-1];