GameSentenceMiner 2.16.1__tar.gz → 2.16.3__tar.gz

{gamesentenceminer-2.16.1 → gamesentenceminer-2.16.3}/GameSentenceMiner/config_gui.py

@@ -391,9 +391,10 @@ class ConfigApp:
         self.use_canned_context_prompt_value = tk.BooleanVar(value=self.settings.ai.use_canned_context_prompt)
         self.ai_dialogue_context_length_value = tk.StringVar(value=str(self.settings.ai.dialogue_context_length))
         
-        # WIP Settings
+        # Overlay Settings
         self.overlay_websocket_port_value = tk.StringVar(value=str(self.settings.overlay.websocket_port))
         self.overlay_websocket_send_value = tk.BooleanVar(value=self.settings.overlay.monitor_to_capture)
+        self.overlay_engine_value = tk.StringVar(value=self.settings.overlay.engine)
         
         # Master Config Settings
         self.switch_to_default_if_not_found_value = tk.BooleanVar(value=self.master_config.switch_to_default_if_not_found)
@@ -623,7 +624,8 @@ class ConfigApp:
             ),
             overlay=Overlay(
                 websocket_port=int(self.overlay_websocket_port_value.get()),
-                monitor_to_capture=self.overlay_monitor.current() if self.monitors else 0
+                monitor_to_capture=self.overlay_monitor.current() if self.monitors else 0,
+                engine=OverlayEngine(self.overlay_engine_value.get()).value if self.overlay_engine_value.get() else OverlayEngine.LENS.value
             )
             # wip=WIP(
             #     overlay_websocket_port=int(self.overlay_websocket_port_value.get()),
@@ -2221,7 +2223,17 @@ class ConfigApp:
         self.overlay_monitor.current(0)
         self.overlay_monitor.config(state="disabled")
         self.current_row += 1
-        
+
+        # Overlay Engine Selection
+        overlay_engine_i18n = overlay_i18n.get('overlay_engine', {})
+        HoverInfoLabelWidget(overlay_frame, text=overlay_engine_i18n.get('label', '...'),
+                             tooltip=overlay_engine_i18n.get('tooltip', '...'),
+                             row=self.current_row, column=0)
+        self.overlay_engine = ttk.Combobox(overlay_frame, values=[e.value for e in OverlayEngine], state="readonly",
+                                           textvariable=self.overlay_engine_value)
+        self.overlay_engine.grid(row=self.current_row, column=1, sticky='EW', pady=2)
+        self.current_row += 1
+
         if self.monitors:
             # Ensure the index is valid
             monitor_index = self.settings.overlay.monitor_to_capture

{gamesentenceminer-2.16.1 → gamesentenceminer-2.16.3}/GameSentenceMiner/gsm.py

@@ -790,7 +790,6 @@ async def async_main(reloading=False):
 
 
 def main():
-    """Main function to run the Game Sentence Miner."""
     logger.info("Starting GSM")
     try:
         asyncio.run(async_main())

{gamesentenceminer-2.16.1 → gamesentenceminer-2.16.3}/GameSentenceMiner/locales/en_us.json

@@ -568,6 +568,10 @@
             "label": "Monitor to Capture:",
             "tooltip": "Select the monitor to capture (1-based index).",
             "not_detected": "OwOCR Not Detected"
+          },
+          "overlay_engine": {
+            "label": "Overlay Engine:",
+            "tooltip": "Select the OCR engine for the overlay. If you use lens, and are on windows, it will use OneOCR to optimize the scan."
           }
         },
         "wip": {

{gamesentenceminer-2.16.1 → gamesentenceminer-2.16.3}/GameSentenceMiner/locales/ja_jp.json

@@ -567,6 +567,10 @@
             "label": "キャプチャ対象モニター:",
             "tooltip": "キャプチャするモニターを選択（1から始まるインデックス）。",
             "not_detected": "OwOCRが検出されません"
+          },
+          "overlay_engine": {
+            "label": "オーバーレイエンジン:",
+            "tooltip": "オーバーレイのOCRエンジンを選択します。Lensを使用していてWindowsの場合、スキャンを最適化するためにOneOCRを使用します。"
           }
         },
         "wip": {

{gamesentenceminer-2.16.1 → gamesentenceminer-2.16.3}/GameSentenceMiner/locales/zh_cn.json

@@ -556,6 +556,10 @@
             "label": "捕获的显示器:",
             "tooltip": "选择要捕获的显示器（从1开始的索引）。",
             "not_detected": "未检测到 OwOCR"
+          },
+          "overlay_engine": {
+            "label": "覆盖层引擎:",
+            "tooltip": "为覆盖层选择 OCR 引擎。如果您使用的是 lens，并且在 windows 上，它将使用 OneOCR 来优化扫描。"
           }
         },
         "wip": {

{gamesentenceminer-2.16.1 → gamesentenceminer-2.16.3}/GameSentenceMiner/util/configuration.py

@@ -630,10 +630,15 @@ class Ai:
             self.gemini_model = 'gemini-2.5-flash-lite'
 
 
+class OverlayEngine(str, Enum):
+    LENS = 'lens'
+    ONEOCR = 'oneocr'
+
 @dataclass_json
 @dataclass
 class Overlay:
     websocket_port: int = 55499
+    engine: str = OverlayEngine.LENS.value
     monitor_to_capture: int = 0
 
     def __post_init__(self):

{gamesentenceminer-2.16.1 → gamesentenceminer-2.16.3}/GameSentenceMiner/util/db.py

@@ -2,6 +2,7 @@
 
 import json
 import os
+import shutil
 import sqlite3
 from sys import platform
 import time
@@ -10,6 +11,7 @@ import threading
 
 from GameSentenceMiner.util.text_log import GameLine
 from GameSentenceMiner.util.configuration import logger, is_dev
+import gzip
 
 
 class SQLiteDB:
@@ -399,6 +401,37 @@ def get_db_directory():
     return os.path.join(config_dir, 'gsm.db')
 
 
+# Backup and compress the database on load, with today's date, up to 5 days ago (clean up old backups)
+def backup_db(db_path: str):
+    backup_dir = os.path.join(os.path.dirname(db_path), "backup", "database")
+    os.makedirs(backup_dir, exist_ok=True)
+    today = time.strftime("%Y-%m-%d")
+    backup_file = os.path.join(backup_dir, f"gsm_{today}.db.gz")
+    
+    # Test, remove backups older than 60 minutes
+    # cutoff = time.time() - 60 * 60
+    # Clean up backups older than 5 days
+    cutoff = time.time() - 5 * 24 * 60 * 60
+    for fname in os.listdir(backup_dir):
+        fpath = os.path.join(backup_dir, fname)
+        if fname.startswith("gsm_") and fname.endswith(".db.gz"):
+            try:
+                file_time = os.path.getmtime(fpath)
+                if file_time < cutoff:
+                    os.remove(fpath)
+                    logger.info(f"Old backup removed: {fpath}")
+            except Exception as e:
+                logger.warning(f"Failed to remove old backup {fpath}: {e}")
+
+    # Create backup if not already present for today
+    if not os.path.exists(backup_file):
+        with open(db_path, "rb") as f_in, open(backup_file, "wb") as f_out:
+            with gzip.GzipFile(fileobj=f_out, mode="wb") as gz_out:
+                shutil.copyfileobj(f_in, gz_out)
+        logger.info(f"Database backup created: {backup_file}")
+
+backup_db(get_db_directory())
+
 gsm_db = SQLiteDB(get_db_directory())
 
 for cls in [AIModelsTable, GameLinesTable]:

{gamesentenceminer-2.16.1 → gamesentenceminer-2.16.3}/GameSentenceMiner/util/get_overlay_coords.py

@@ -1,19 +1,67 @@
 import asyncio
 import io
 import base64
+import json
 import math
 import os
 import time
 from PIL import Image
 from typing import Dict, Any, List, Tuple
+import json
+from rapidfuzz.distance import Levenshtein
 
 # Local application imports
 from GameSentenceMiner.ocr.gsm_ocr_config import set_dpi_awareness
-from GameSentenceMiner.util.configuration import get_config, is_windows
+from GameSentenceMiner.util.configuration import OverlayEngine, get_config, is_windows, is_beangate, logger
 from GameSentenceMiner.util.electron_config import get_ocr_language
-from GameSentenceMiner.obs import get_screenshot_PIL, logger
+from GameSentenceMiner.obs import get_screenshot_PIL
 from GameSentenceMiner.web.texthooking_page import send_word_coordinates_to_overlay
 
+# def align_and_correct(ocr_json, reference_text):
+#     logger.info(f"Starting align_and_correct with reference_text: '{reference_text}'")
+#     corrected = []
+#     ref_chars = list(reference_text)
+#     logger.info(f"Reference chars: {ref_chars}")
+
+#     for block_idx, block in enumerate(ocr_json):
+#         logger.info(f"Processing block {block_idx}: {block}")
+#         ocr_chars = [w["text"] for w in block["words"]]
+#         ocr_str = "".join(ocr_chars)
+
+#         # Compute edit operations from OCR → Reference
+#         ops = Levenshtein.editops(ocr_str, "".join(ref_chars))
+
+#         corrected_words = block["words"][:]
+
+#         # Apply corrections
+#         for op_idx, (op, i, j) in enumerate(ops):
+#             logger.info(f"Operation {op_idx}: {op}, i={i}, j={j}")
+#             if op == "replace":
+#                 logger.info(f"Replacing word at index {i} ('{corrected_words[i]['text']}') with reference char '{ref_chars[j]}'")
+#                 corrected_words[i]["text"] = ref_chars[j]
+#             elif op == "insert":
+#                 if i > 0:
+#                     prev = corrected_words[i - 1]["bounding_rect"]
+#                     bbox = prev  # simple: copy neighbor bbox
+#                 else:
+#                     bbox = corrected_words[0]["bounding_rect"]
+#                 corrected_words.insert(i, {
+#                     "text": ref_chars[j],
+#                     "bounding_rect": bbox,
+#                     "confidence": 1.0
+#                 })
+#             elif op == "delete":
+#                 logger.info(f"Deleting word at index {i} ('{corrected_words[i]['text']}')")
+#                 corrected_words[i]["text"] = ""  # mark empty
+
+#         corrected_words = [w for w in corrected_words if w["text"]]
+
+#         block["words"] = corrected_words
+#         block["text"] = "".join(w["text"] for w in corrected_words)
+#         corrected.append(block)
+
+#     return corrected
+
 # Conditionally import OCR engines
 try:
     if os.path.exists(os.path.expanduser('~/.config/oneocr/oneocr.dll')):
@@ -218,6 +266,15 @@ class OverlayProcessor:
             )
         else:
             composite_image = full_screenshot
+            
+        # If User Home is beangate
+        if is_beangate:
+            with open("oneocr_results.json", "w", encoding="utf-8") as f:
+                f.write(json.dumps(oneocr_results, ensure_ascii=False, indent=2))
+            
+        if get_config().overlay.engine == OverlayEngine.ONEOCR.value and self.oneocr:
+            logger.info("Using OneOCR results for overlay as configured.")
+            return oneocr_results
         
         # 4. Use Google Lens on the cleaner composite image for higher accuracy
         res = self.lens(

{gamesentenceminer-2.16.1 → gamesentenceminer-2.16.3}/GameSentenceMiner/vad.py

@@ -202,7 +202,7 @@ class WhisperVADProcessor(VADProcessor):
                     logger.info(
                         "Unknown single character segment, not skipping, but logging, please report if this is a mistake: " + segment.text)
                     
-            if segment.no_speech_prob and segment.no_speech_prob > 0.6:
+            if segment.no_speech_prob and segment.no_speech_prob > 0.9:
                 logger.debug(f"Skipping segment with high no_speech_prob: {segment.no_speech_prob} for segment {segment.text} at {segment.start}-{segment.end}")
                 continue
             
@@ -211,7 +211,7 @@ class WhisperVADProcessor(VADProcessor):
                 logger.debug(f"Skipping segment with low unique words: {unique_words} for segment {segment.text} at {segment.start}-{segment.end}")
                 continue
                 
-            if previous_segment and segment.start - previous_segment.end > 5 and segment.no_speech_prob > .3:
+            if segment.seek > 0 and segment.no_speech_prob > .3:
                 logger.debug(f"Skipping segment after long pause with high no_speech_prob after: {segment.no_speech_prob} for segment {segment.text} at {segment.start}-{segment.end}")
                 continue
 

{gamesentenceminer-2.16.1 → gamesentenceminer-2.16.3}/GameSentenceMiner/web/database_api.py

@@ -1,5 +1,7 @@
 import datetime
 import re
+import csv
+import io
 from collections import defaultdict
 
 import flask
@@ -135,8 +137,8 @@ def register_database_api_routes(app):
                     'date_range': f"{min_date.strftime('%Y-%m-%d')} to {max_date.strftime('%Y-%m-%d')}" if min_date != max_date else min_date.strftime('%Y-%m-%d')
                 })
             
-            # Sort by first entry date (most recent first)
-            games_data.sort(key=lambda x: x['first_entry_date'], reverse=True)
+            # Sort by total characters (most characters first)
+            games_data.sort(key=lambda x: x['total_characters'], reverse=True)
             
             return jsonify({'games': games_data}), 200
             
@@ -732,7 +734,8 @@ def register_database_api_routes(app):
             color = colors[i % len(colors)]
             
             datasets.append({
-                "label": f"{game} - Lines Received",
+                "label": f"{game}",
+                "for": "Lines Received",
                 "data": final_data[game]['lines'],
                 "borderColor": color,
                 "backgroundColor": f"{color}33", # Semi-transparent for fill
@@ -740,7 +743,8 @@ def register_database_api_routes(app):
                 "tension": 0.1
             })
             datasets.append({
-                "label": f"{game} - Characters Read",
+                "label": f"{game}",
+                "for": "Characters Read",
                 "data": final_data[game]['chars'],
                 "borderColor": color,
                 "backgroundColor": f"{color}33",
@@ -780,4 +784,165 @@ def register_database_api_routes(app):
             "currentGameStats": current_game_stats,
             "allGamesStats": all_games_stats,
             "allLinesData": all_lines_data
-        })
+        })
+
+    @app.route('/api/import-exstatic', methods=['POST'])
+    def api_import_exstatic():
+        """
+        Import ExStatic CSV data into GSM database.
+        Expected CSV format: uuid,given_identifier,name,line,time
+        """
+        try:
+            # Check if file is provided
+            if 'file' not in request.files:
+                return jsonify({'error': 'No file provided'}), 400
+            
+            file = request.files['file']
+            if file.filename == '':
+                return jsonify({'error': 'No file selected'}), 400
+            
+            # Validate file type
+            if not file.filename.lower().endswith('.csv'):
+                return jsonify({'error': 'File must be a CSV file'}), 400
+            
+            # Read and parse CSV
+            try:
+                # Read file content as text with proper encoding handling
+                file_content = file.read().decode('utf-8-sig')  # Handle BOM if present
+                
+                # First, get the header line manually to avoid issues with multi-line content
+                lines = file_content.split('\n')
+                if len(lines) == 1 and not lines[0].strip():
+                    return jsonify({'error': 'Empty CSV file'}), 400
+                
+                header_line = lines[0].strip()
+                logger.info(f"Header line: {header_line}")
+                
+                # Parse headers manually
+                header_reader = csv.reader([header_line])
+                try:
+                    headers = next(header_reader)
+                    headers = [h.strip() for h in headers]  # Clean whitespace
+                    logger.info(f"Parsed headers: {headers}")
+                except StopIteration:
+                    return jsonify({'error': 'Could not parse CSV headers'}), 400
+                
+                # Validate headers
+                expected_headers = {'uuid', 'given_identifier', 'name', 'line', 'time'}
+                actual_headers = set(headers)
+                
+                if not expected_headers.issubset(actual_headers):
+                    missing_headers = expected_headers - actual_headers
+                    # Check if this looks like a stats CSV instead of lines CSV
+                    if 'client' in actual_headers and 'chars_read' in actual_headers:
+                        return jsonify({
+                            'error': 'This appears to be an ExStatic stats CSV. Please upload the ExStatic lines CSV file instead. The lines CSV should contain columns: uuid, given_identifier, name, line, time'
+                        }), 400
+                    else:
+                        return jsonify({
+                            'error': f'Invalid CSV format. Missing required columns: {", ".join(missing_headers)}. Expected format: uuid, given_identifier, name, line, time. Found headers: {", ".join(actual_headers)}'
+                        }), 400
+                
+                # Now parse the full CSV with proper handling for multi-line fields
+                file_io = io.StringIO(file_content)
+                csv_reader = csv.DictReader(file_io, quoting=csv.QUOTE_MINIMAL, skipinitialspace=True)
+                
+                # Process CSV rows
+                imported_lines = []
+                games_set = set()
+                errors = []
+                seen_uuids = set()  # Track UUIDs within this import batch
+                
+                for row_num, row in enumerate(csv_reader):
+                    try:
+                        # Extract and validate required fields
+                        uuid = row.get('uuid', '').strip()
+                        name = row.get('name', '').strip()
+                        line = row.get('line', '').strip()
+                        time_str = row.get('time', '').strip()
+                        
+                        # Validate required fields
+                        if not uuid:
+                            errors.append(f"Row {row_num}: Missing UUID")
+                            continue
+                        if not name:
+                            errors.append(f"Row {row_num}: Missing name")
+                            continue
+                        if not line:
+                            errors.append(f"Row {row_num}: Missing line text")
+                            continue
+                        if not time_str:
+                            errors.append(f"Row {row_num}: Missing time")
+                            continue
+                        
+                        # Check for duplicates within this import batch
+                        if uuid in seen_uuids:
+                            logger.info(f"Skipping duplicate UUID within import batch: {uuid}")
+                            continue
+                        seen_uuids.add(uuid)
+                        
+                        # Convert time to timestamp
+                        try:
+                            timestamp = float(time_str)
+                        except ValueError:
+                            errors.append(f"Row {row_num}: Invalid time format: {time_str}")
+                            continue
+                        
+                        # Clean up line text (remove extra whitespace and newlines)
+                        line_text = line.strip()
+                        
+                        # Check if this UUID already exists in database
+                        existing_line = GameLinesTable.get(uuid)
+                        if existing_line:
+                            logger.info(f"Skipping duplicate UUID already in database: {uuid}")
+                            continue
+                        
+                        # Create GameLinesTable entry
+                        game_line = GameLinesTable(
+                            id=uuid,
+                            game_name=name,
+                            line_text=line_text,
+                            timestamp=timestamp
+                        )
+                        
+                        imported_lines.append(game_line)
+                        games_set.add(name)
+                        
+                    except Exception as e:
+                        errors.append(f"Row {row_num}: Error processing row - {str(e)}")
+                        continue
+                
+                # Import lines into database
+                imported_count = 0
+                for game_line in imported_lines:
+                    try:
+                        game_line.add()
+                        imported_count += 1
+                    except Exception as e:
+                        logger.error(f"Failed to import line {game_line.id}: {e}")
+                        errors.append(f"Failed to import line {game_line.id}: {str(e)}")
+                
+                # Prepare response
+                response_data = {
+                    'message': f'Successfully imported {imported_count} lines from {len(games_set)} games',
+                    'imported_count': imported_count,
+                    'games_count': len(games_set),
+                    'games': list(games_set)
+                }
+                
+                if errors:
+                    response_data['warnings'] = errors
+                    response_data['warning_count'] = len(errors)
+                
+                logger.info(f"ExStatic import completed: {imported_count} lines from {len(games_set)} games")
+                
+                return jsonify(response_data), 200
+                
+            except csv.Error as e:
+                return jsonify({'error': f'CSV parsing error: {str(e)}'}), 400
+            except UnicodeDecodeError:
+                return jsonify({'error': 'File encoding error. Please ensure the CSV is UTF-8 encoded.'}), 400
+            
+        except Exception as e:
+            logger.error(f"Error in ExStatic import: {e}")
+            return jsonify({'error': f'Import failed: {str(e)}'}), 500

{gamesentenceminer-2.16.1 → gamesentenceminer-2.16.3}/GameSentenceMiner/web/static/js/shared.js

@@ -491,6 +491,70 @@ function escapeRegex(string) {
     return string.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
 }
 
+// Screenshot functionality
+function initializeScreenshotButton() {
+    const screenshotButton = document.getElementById('screenshotToggle');
+    
+    if (!screenshotButton) {
+        return; // Screenshot button not available on this page
+    }
+    
+    screenshotButton.addEventListener('click', takeScreenshot);
+}
+
+async function takeScreenshot() {
+    try {
+        // Check if html2canvas is available
+        if (typeof html2canvas === 'undefined') {
+            console.error('html2canvas library not loaded');
+            return;
+        }
+        
+        // Generate timestamp for filename
+        const now = new Date();
+        const timestamp = now.getFullYear() + '-' +
+                         String(now.getMonth() + 1).padStart(2, '0') + '-' +
+                         String(now.getDate()).padStart(2, '0') + '_' +
+                         String(now.getHours()).padStart(2, '0') + '-' +
+                         String(now.getMinutes()).padStart(2, '0') + '-' +
+                         String(now.getSeconds()).padStart(2, '0');
+        
+        const filename = `screenshot_${timestamp}.png`;
+        
+        // Capture the entire page
+        const canvas = await html2canvas(document.body, {
+            useCORS: true,
+            allowTaint: true,
+            scale: 1,
+            scrollX: 0,
+            scrollY: 0,
+            width: document.body.scrollWidth,
+            height: document.body.scrollHeight
+        });
+        
+        // Convert canvas to blob
+        canvas.toBlob(function(blob) {
+            // Create download link
+            const link = document.createElement('a');
+            link.download = filename;
+            link.href = URL.createObjectURL(blob);
+            
+            // Trigger download
+            document.body.appendChild(link);
+            link.click();
+            document.body.removeChild(link);
+            
+            // Clean up the URL object after a short delay to avoid race condition
+            setTimeout(function() {
+                URL.revokeObjectURL(link.href);
+            }, 100);
+        }, 'image/png');
+        
+    } catch (error) {
+        console.error('Screenshot failed:', error);
+    }
+}
+
 // Initialize shared functionality when DOM loads
 document.addEventListener('DOMContentLoaded', function() {
     // Initialize theme toggle
@@ -499,6 +563,9 @@ document.addEventListener('DOMContentLoaded', function() {
     // Initialize modal handlers
     initializeModalHandlers();
     
+    // Initialize screenshot button
+    initializeScreenshotButton();
+    
     // Initialize settings manager if settings toggle exists
     if (document.getElementById('settingsToggle')) {
         new SettingsManager();