GameSentenceMiner 2.16.7__py3-none-any.whl → 2.16.9__py3-none-any.whl

GameSentenceMiner/util/db.py

@@ -1,16 +1,22 @@
 
 
+from datetime import datetime
 import json
 import os
+import random
 import shutil
 import sqlite3
 from sys import platform
 import time
 from typing import Any, Dict, List, Optional, Tuple, Union, Type, TypeVar
 import threading
+import uuid
+
+import pytz
+from datetime import timedelta
 
 from GameSentenceMiner.util.text_log import GameLine
-from GameSentenceMiner.util.configuration import logger, is_dev
+from GameSentenceMiner.util.configuration import get_stats_config, logger, is_dev
 import gzip
 
 
@@ -23,6 +29,14 @@ class SQLiteDB:
     def __init__(self, db_path: str):
         self.db_path = db_path
         self._lock = threading.Lock()
+        
+    def backup(self, backup_path: str):
+        """ Create a backup of the database using built in SQLite backup API. """
+        with self._lock, sqlite3.connect(self.db_path, check_same_thread=False) as conn:
+            with sqlite3.connect(backup_path, check_same_thread=False) as backup_conn:
+                conn.backup(backup_conn)
+                if is_dev:
+                    logger.debug(f"Database backed up to {backup_path}")
 
     def execute(self, query: str, params: Union[Tuple, Dict] = (), commit: bool = False) -> sqlite3.Cursor:
         with self._lock, sqlite3.connect(self.db_path, check_same_thread=False) as conn:
@@ -341,7 +355,7 @@ class GameLinesTable(SQLiteDBTable):
         self.game_name = game_name
         self.line_text = line_text
         self.context = context
-        self.timestamp = timestamp if timestamp is not None else time.time()
+        self.timestamp = timestamp if timestamp is not None else datetime.now().timestamp()
         self.screenshot_in_anki = screenshot_in_anki if screenshot_in_anki is not None else ''
         self.audio_in_anki = audio_in_anki if audio_in_anki is not None else ''
         self.screenshot_path = screenshot_path if screenshot_path is not None else ''
@@ -388,6 +402,20 @@ class GameLinesTable(SQLiteDBTable):
         # logger.info("Adding GameLine to DB: %s", new_line)
         new_line.add()
         return new_line
+    
+    @classmethod
+    def add_lines(cls, gamelines: List[GameLine]):
+        new_lines = [cls(id=gl.id, game_name=gl.scene,
+                         line_text=gl.text, timestamp=gl.time.timestamp()) for gl in gamelines]
+        # logger.info("Adding %d GameLines to DB", len(new_lines))
+        params = [(line.id, line.game_name, line.line_text, line.timestamp, line.screenshot_in_anki,
+                   line.audio_in_anki, line.screenshot_path, line.audio_path, line.replay_path, line.translation)
+                  for line in new_lines]
+        cls._db.executemany(
+            f"INSERT INTO {cls._table} (id, game_name, line_text, timestamp, screenshot_in_anki, audio_in_anki, screenshot_path, audio_path, replay_path, translation) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
+            params,
+            commit=True
+        )
 
 
 def get_db_directory():
@@ -441,3 +469,56 @@ for cls in [AIModelsTable, GameLinesTable]:
     # Uncomment to start fresh every time
     # cls.drop()
     # cls.set_db(gsm_db)  # --- IGNORE ---
+    
+# import random
+# import uuid
+# from datetime import datetime
+# from GameSentenceMiner.util.text_log import GameLine
+# from GameSentenceMiner.util.db import GameLinesTable
+
+# List of common Japanese characters (kanji, hiragana, katakana)
+# japanese_chars = (
+#     "あいうえおかきくけこさしすせそたちつてとなにぬねのはひふへほまみむめもやゆよらりるれろわをん"
+#     "アイウエオカキクケコサシスセソタチツテトナニヌネノハヒフヘホマミムメモヤユヨラリルレロワヲン"
+#     "亜唖娃阿哀愛挨悪握圧扱宛嵐安暗案闇以衣位囲医依委威為胃尉異移維緯"
+#     # ... (add more kanji for more variety)
+# )
+
+# def random_japanese_text(length):
+#     return ''.join(random.choices(japanese_chars, k=length))
+
+# batch_size = 1000
+# lines_batch = []
+
+# def random_datetime(start_year=2024, end_year=2026):
+#     start = datetime(start_year, 1, 1, 0, 0, 0)
+#     end = datetime(end_year, 12, 31, 23, 59, 59)
+#     delta = end - start
+#     random_seconds = random.randint(0, int(delta.total_seconds()))
+#     return start + timedelta(seconds=random_seconds)
+
+# from datetime import timedelta
+
+# for i in range(500000):  # Adjust for desired number of lines
+#     line_text = random_japanese_text(random.randint(25, 40))
+#     lines_batch.append(GameLine(
+#         id=str(uuid.uuid1()),
+#         text=line_text,
+#         time=random_datetime(),
+#         prev=None,
+#         next=None,
+#         index=i,
+#         scene="RandomScene"
+#     ))
+    
+#     if len(lines_batch) >= batch_size:
+#         GameLinesTable.add_lines(lines_batch)
+#         lines_batch = []
+#     if i % 1000 == 0:
+#         print(f"Inserted {i} lines...")
+
+# # Insert any remaining lines
+# if lines_batch:
+#     GameLinesTable.add_lines(lines_batch)
+
+print("Done populating GameLinesDB with random Japanese text.")

GameSentenceMiner/util/ffmpeg.py

@@ -495,6 +495,7 @@ def get_video_timings(video_path, game_line, anki_card_creation_time=None):
     total_seconds = file_length - time_delta.total_seconds()
     total_seconds_after_offset = total_seconds + get_config().audio.beginning_offset
     if total_seconds < 0 or total_seconds >= file_length:
+        logger.debug(f"get_video_timings: file_mod_time={file_mod_time}, file_length={file_length}, time_delta={time_delta}, total_seconds={total_seconds}, total_seconds_after_offset={total_seconds_after_offset}")
         logger.error("Line mined is outside of the replay buffer! Defaulting to the last 30 seconds of the replay buffer.")
         logger.info("Recommend either increasing replay buffer length in OBS Settings or mining faster.")
         return max(file_length - 30, 0), 0, max(file_length - 30, 0), file_length

GameSentenceMiner/util/get_overlay_coords.py

@@ -4,6 +4,7 @@ import base64
 import json
 import math
 import os
+import threading
 import time
 from PIL import Image
 from typing import Dict, Any, List, Tuple
@@ -16,6 +17,7 @@ from GameSentenceMiner.util.configuration import OverlayEngine, get_config, is_w
 from GameSentenceMiner.util.electron_config import get_ocr_language
 from GameSentenceMiner.obs import get_screenshot_PIL
 from GameSentenceMiner.web.texthooking_page import send_word_coordinates_to_overlay
+from GameSentenceMiner.web.gsm_websocket import overlay_server_thread
 
 # def align_and_correct(ocr_json, reference_text):
 #     logger.info(f"Starting align_and_correct with reference_text: '{reference_text}'")
@@ -80,6 +82,32 @@ try:
     import mss
 except ImportError:
     mss = None
+    
+class OverlayThread(threading.Thread):
+    """
+    A thread to run the overlay processing loop.
+    This is a simple wrapper around asyncio to run the overlay processing
+    in a separate thread.
+    """
+    def __init__(self):
+        super().__init__()
+        self.overlay_processor = OverlayProcessor()
+        self.loop = asyncio.new_event_loop()
+        self.daemon = True  # Ensure thread exits when main program exits
+
+    def run(self):
+        """Runs the overlay processing loop."""
+        asyncio.set_event_loop(self.loop)
+        self.loop.run_until_complete(self.overlay_loop())
+
+    async def overlay_loop(self):
+        """Main loop to periodically process and send overlay data."""
+        while True:
+            if get_config().overlay.periodic and overlay_server_thread.has_clients():
+                await self.overlay_processor.find_box_and_send_to_overlay('')
+                await asyncio.sleep(get_config().overlay.periodic_interval)  # Adjust the interval as needed
+            else:
+                await asyncio.sleep(3)  # Sleep briefly when not active
 
 class OverlayProcessor:
     """
@@ -254,7 +282,7 @@ class OverlayProcessor:
                 return_coords=True,
                 multiple_crop_coords=True,
                 return_one_box=False,
-                furigana_filter_sensitivity=None # Disable furigana filtering
+                furigana_filter_sensitivity=None, # Disable furigana filtering
             )
 
             # 3. Create a composite image with only the detected text regions

GameSentenceMiner/vad.py

@@ -1,6 +1,7 @@
 import tempfile
 import time
 import warnings
+import requests
 from abc import abstractmethod, ABC
 
 from GameSentenceMiner.util import configuration, ffmpeg
@@ -37,19 +38,7 @@ class VADSystem:
             if not result.success and get_config().vad.backup_vad_model != configuration.OFF:
                 logger.info("No voice activity detected, using backup VAD model.")
                 result = self._do_vad_processing(get_config().vad.backup_vad_model, input_audio, output_audio, game_line)
-            if not result.success:
-                if get_config().vad.add_audio_on_no_results:
-                    logger.info("No voice activity detected, using full audio.")
-                    result.output_audio = input_audio
-                else:
-                    logger.info("No voice activity detected.")
-                    return result
-            else:
-                logger.info(result.trim_successful_string())
             return result
-        else:
-            return VADResult(True, 0, get_audio_length(input_audio), "OFF", [], input_audio)
-
 
     def _do_vad_processing(self, model, input_audio, output_audio, game_line):
         match model:
@@ -185,7 +174,7 @@ class WhisperVADProcessor(VADProcessor):
         # Transcribe the audio using Whisper
         with warnings.catch_warnings():
             warnings.simplefilter("ignore")
-            result: WhisperResult = self.vad_model.transcribe(temp_wav, vad=True, language=get_config().vad.language,
+            result: WhisperResult = self.vad_model.transcribe(temp_wav, vad=True, language=get_config().vad.language, vad_filter=get_config().vad.use_vad_filter_for_whisper,
                                                              temperature=0.0)
         voice_activity = []
 
@@ -406,6 +395,7 @@ def test_vad_processors():
     get_config().vad.cut_and_splice_segments = False
     get_config().vad.trim_beginning = True
     get_config().vad.add_audio_on_no_results = True
+    get_config().vad.use_vad_filter_for_whisper = False
     for processor, out_name in processors:
         logger.info("Testing Trim Audio with " + processor.vad_system_name)
         out_path = os.path.join(output_dir, out_name.replace("after_splice_", "after_trim_"))

GameSentenceMiner/web/database_api.py

@@ -12,6 +12,7 @@ import regex
 
 from GameSentenceMiner.util.db import GameLinesTable
 from GameSentenceMiner.util.configuration import get_stats_config, logger, get_config, save_current_config, save_stats_config
+from GameSentenceMiner.util.text_log import GameLine
 from GameSentenceMiner.web.stats import (
     calculate_kanji_frequency, calculate_heatmap_data, calculate_total_chars_per_game,
     calculate_reading_time_per_game, calculate_reading_speed_per_game,
@@ -304,7 +305,7 @@ def register_database_api_routes(app):
                 'streak_requirement_hours': config.streak_requirement_hours,
                 'reading_hours_target': config.reading_hours_target,
                 'character_count_target': config.character_count_target,
-                'games_target': config.games_target
+                'games_target': config.games_target,
             }), 200
         except Exception as e:
             logger.error(f"Error getting settings: {e}")
@@ -806,6 +807,8 @@ def register_database_api_routes(app):
         daily_data = defaultdict(lambda: defaultdict(lambda: {'lines': 0, 'chars': 0}))
         
         
+        
+        
         # start_time = time.perf_counter()
         # for line in all_lines:
         #     day_str = datetime.date.fromtimestamp(float(line.timestamp)).strftime('%Y-%m-%d')
@@ -975,11 +978,15 @@ def register_database_api_routes(app):
                 csv_reader = csv.DictReader(file_io, quoting=csv.QUOTE_MINIMAL, skipinitialspace=True)
                 
                 # Process CSV rows
-                imported_lines = []
                 games_set = set()
                 errors = []
-                seen_uuids = set()  # Track UUIDs + Line within import batch
-
+                
+                all_lines = GameLinesTable.all()
+                existing_uuids = {line.id for line in all_lines}
+                batch_size = 1000  # For logging progress
+                batch_insert = []
+                imported_count = 0
+                
                 def get_line_hash(uuid: str, line_text: str) -> str:
                     return uuid + '|' + line_text.strip()
 
@@ -1006,10 +1013,11 @@ def register_database_api_routes(app):
                             continue
                         
                         line_hash = get_line_hash(game_uuid, line)
-                        if line_hash in seen_uuids:
-                            logger.info(f"Skipping duplicate line from game UUID {game_uuid} in import batch")
+                        
+                        # Check if this line already exists in database
+                        if line_hash in existing_uuids:
+                            logger.info(f"Skipping duplicate UUID already in database: {line_hash}")
                             continue
-                        seen_uuids.add(line_hash)
 
                         # Convert time to timestamp
                         try:
@@ -1021,37 +1029,52 @@ def register_database_api_routes(app):
                         # Clean up line text (remove extra whitespace and newlines)
                         line_text = line.strip()
                         
-                        # Check if this line already exists in database
-                        existing_line = GameLinesTable.get(line_hash)
-                        if existing_line:
-                            logger.info(f"Skipping duplicate UUID already in database: {line_hash}")
-                            continue
-                        
                         # Create GameLinesTable entry
-                        game_line = GameLinesTable(
+                        # Convert timestamp float to datetime object
+                        dt = datetime.datetime.fromtimestamp(timestamp)
+                        batch_insert.append(GameLine(
                             id=line_hash,
-                            game_name=game_name,
-                            line_text=line_text,
-                            timestamp=timestamp
-                        )
+                            text=line_text,
+                            scene=game_name,
+                            time=dt,
+                            prev=None,
+                            next=None,
+                            index=0,
+                        ))
+                        
+                        logger.info(f"Batch insert size: {len(batch_insert)}")
                         
-                        imported_lines.append(game_line)
+                        existing_uuids.add(line_hash)  # Add to existing to prevent duplicates in same import
+                        
+                        if len(batch_insert) >= batch_size:
+                            logger.info(f"Importing batch of {len(batch_insert)} lines...")
+                            GameLinesTable.add_lines(batch_insert)
+                            imported_count += len(batch_insert)
+                            batch_insert = []
                         games_set.add(game_name)
                         
                     except Exception as e:
+                        logger.error(f"Error processing row {row_num}: {e}")
                         errors.append(f"Row {row_num}: Error processing row - {str(e)}")
                         continue
+                    
+                # Insert the rest of the batch
+                if batch_insert:
+                    logger.info(f"Importing final batch of {len(batch_insert)} lines...")
+                    GameLinesTable.add_lines(batch_insert)
+                    imported_count += len(batch_insert)
+                    batch_insert = []
                 
-                # Import lines into database
-                imported_count = 0
-                for game_line in imported_lines:
-                    try:
-                        game_line.add()
-                        imported_count += 1
-                    except Exception as e:
-                        logger.error(f"Failed to import line {game_line.id}: {e}")
-                        errors.append(f"Failed to import line {game_line.id}: {str(e)}")
-                
+                # # Import lines into database
+                # imported_count = 0
+                # for game_line in imported_lines:
+                #     try:
+                #         game_line.add()
+                #         imported_count += 1
+                #     except Exception as e:
+                #         logger.error(f"Failed to import line {game_line.id}: {e}")
+                #         errors.append(f"Failed to import line {game_line.id}: {str(e)}")
+
                 # Prepare response
                 response_data = {
                     'message': f'Successfully imported {imported_count} lines from {len(games_set)} games',
@@ -1066,6 +1089,8 @@ def register_database_api_routes(app):
                 
                 logger.info(f"ExStatic import completed: {imported_count} lines from {len(games_set)} games")
                 
+                logger.info(f"Import response: {response_data}")
+                
                 return jsonify(response_data), 200
                 
             except csv.Error as e:

GameSentenceMiner/web/static/js/search.js

@@ -186,7 +186,7 @@ class SentenceSearchApp {
         
         // Format timestamp to ISO format
         const date = new Date(result.timestamp * 1000);
-        const formattedDate = date.toISOString().split('T')[0] + ' ' + date.toTimeString().split(' ')[0];
+        const formattedDate = `${date.getFullYear()}-${String(date.getMonth() + 1).padStart(2, '0')}-${String(date.getDate()).padStart(2, '0')} ${date.toTimeString().split(' ')[0]}`;
         
         div.innerHTML = `
             <div class="result-sentence">${highlightedText}</div>

GameSentenceMiner/web/static/js/stats.js

@@ -128,7 +128,7 @@ document.addEventListener('DOMContentLoaded', function () {
             for (let day = 0; day < 7; day++) {
                 const date = grid[day][week];
                 if (date) {
-                    const dateStr = date.toISOString().split('T')[0];
+                    const dateStr = `${date.getFullYear()}-${String(date.getMonth() + 1).padStart(2, '0')}-${String(date.getDate()).padStart(2, '0')}`;
                     const activity = yearData[dateStr] || 0;
                     dates.push({ date: dateStr, activity: activity });
                 }
@@ -154,7 +154,8 @@ document.addEventListener('DOMContentLoaded', function () {
         }
         
         // Calculate current streak from today backwards
-        const today = new Date().toISOString().split('T')[0];
+        const date = new Date();
+        const today = `${date.getFullYear()}-${String(date.getMonth() + 1).padStart(2, '0')}-${String(date.getDate()).padStart(2, '0')}`;
         
         // Find today's index or the most recent date before today
         let todayIndex = -1;
@@ -182,7 +183,10 @@ document.addEventListener('DOMContentLoaded', function () {
             // Group timestamps by day for this year
             const dailyTimestamps = {};
             for (const line of allLinesForYear) {
-                const dateStr = new Date(parseFloat(line.timestamp) * 1000).toISOString().split('T')[0];
+                const ts = parseFloat(line.timestamp);
+                if (isNaN(ts)) continue;
+                const dateObj = new Date(ts * 1000);
+                const dateStr = `${dateObj.getFullYear()}-${String(dateObj.getMonth() + 1).padStart(2, '0')}-${String(dateObj.getDate()).padStart(2, '0')}`;
                 if (!dailyTimestamps[dateStr]) {
                     dailyTimestamps[dateStr] = [];
                 }
@@ -326,7 +330,7 @@ document.addEventListener('DOMContentLoaded', function () {
                     
                     const date = grid[day][week];
                     if (date) {
-                        const dateStr = date.toISOString().split('T')[0];
+                        const dateStr = `${date.getFullYear()}-${String(date.getMonth() + 1).padStart(2, '0')}-${String(date.getDate()).padStart(2, '0')}`;
                         const activity = yearData[dateStr] || 0;
                         
                         if (activity > 0 && maxActivity > 0) {
@@ -881,11 +885,14 @@ document.addEventListener('DOMContentLoaded', function () {
             // Group by day and calculate reading time using AFK timer logic
             const dailyTimestamps = {};
             for (const line of recentData) {
-                const dateStr = new Date(line.timestamp * 1000).toISOString().split('T')[0];
+                const ts = parseFloat(line.timestamp);
+                if (isNaN(ts)) continue;
+                const dateObj = new Date(ts * 1000);
+                const dateStr = `${dateObj.getFullYear()}-${String(dateObj.getMonth() + 1).padStart(2, '0')}-${String(dateObj.getDate()).padStart(2, '0')}`;
                 if (!dailyTimestamps[dateStr]) {
                     dailyTimestamps[dateStr] = [];
                 }
-                dailyTimestamps[dateStr].push(line.timestamp);
+                dailyTimestamps[dateStr].push(ts);
             }
             
             for (const [dateStr, timestamps] of Object.entries(dailyTimestamps)) {
@@ -906,14 +913,20 @@ document.addEventListener('DOMContentLoaded', function () {
         } else if (metricType === 'characters') {
             // Group by day and sum characters
             for (const line of recentData) {
-                const dateStr = new Date(line.timestamp * 1000).toISOString().split('T')[0];
+                const ts = parseFloat(line.timestamp);
+                if (isNaN(ts)) continue;
+                const dateObj = new Date(ts * 1000);
+                const dateStr = `${dateObj.getFullYear()}-${String(dateObj.getMonth() + 1).padStart(2, '0')}-${String(dateObj.getDate()).padStart(2, '0')}`;
                 dailyTotals[dateStr] = (dailyTotals[dateStr] || 0) + (line.characters || 0);
             }
         } else if (metricType === 'games') {
             // Group by day and count unique games
             const dailyGames = {};
             for (const line of recentData) {
-                const dateStr = new Date(line.timestamp * 1000).toISOString().split('T')[0];
+                const ts = parseFloat(line.timestamp);
+                if (isNaN(ts)) continue;
+                const dateObj = new Date(ts * 1000);
+                const dateStr = `${dateObj.getFullYear()}-${String(dateObj.getMonth() + 1).padStart(2, '0')}-${String(dateObj.getDate()).padStart(2, '0')}`;
                 if (!dailyGames[dateStr]) {
                     dailyGames[dateStr] = new Set();
                 }
@@ -1141,14 +1154,19 @@ document.addEventListener('DOMContentLoaded', function () {
     function loadDashboardData(data = null) {
         function updateTodayOverview(allLinesData) {
             // Get today's date string (YYYY-MM-DD)
+            // Get today's date string (YYYY-MM-DD), timezone aware (local time)
             const today = new Date();
-            const todayStr = today.toISOString().split('T')[0];
+            const pad = n => n.toString().padStart(2, '0');
+            const todayStr = `${today.getFullYear()}-${pad(today.getMonth() + 1)}-${pad(today.getDate())}`;
             document.getElementById('todayDate').textContent = todayStr;
 
             // Filter lines for today
             const todayLines = (allLinesData || []).filter(line => {
                 if (!line.timestamp) return false;
-                const lineDate = new Date(parseFloat(line.timestamp) * 1000).toISOString().split('T')[0];
+                const ts = parseFloat(line.timestamp);
+                if (isNaN(ts)) return false;
+                const dateObj = new Date(ts * 1000);
+                const lineDate = `${dateObj.getFullYear()}-${pad(dateObj.getMonth() + 1)}-${pad(dateObj.getDate())}`;
                 return lineDate === todayStr;
             });
 
@@ -1195,7 +1213,13 @@ document.addEventListener('DOMContentLoaded', function () {
                 .map(l => parseFloat(l.timestamp))
                 .filter(ts => !isNaN(ts))
                 .sort((a, b) => a - b);
-            const afkTimerSeconds = 120;
+            // Get AFK timer from settings modal if available
+            let afkTimerSeconds = 120; // default
+            const afkTimerInput = document.getElementById('afkTimer');
+            if (afkTimerInput && afkTimerInput.value) {
+                const parsed = parseInt(afkTimerInput.value, 10);
+                if (!isNaN(parsed) && parsed > 0) afkTimerSeconds = parsed;
+            }
             if (timestamps.length >= 2) {
                 for (let i = 1; i < timestamps.length; i++) {
                     const gap = timestamps[i] - timestamps[i-1];