GameSentenceMiner 2.17.0__py3-none-any.whl → 2.17.2__py3-none-any.whl

GameSentenceMiner/util/db.py

@@ -264,6 +264,50 @@ class SQLiteDBTable:
     @classmethod
     def drop(cls):
         cls._db.execute(f"DROP TABLE IF EXISTS {cls._table}", commit=True)
+        
+    @classmethod
+    def has_column(cls, column_name: str) -> bool:
+        row = cls._db.fetchone(
+            f"PRAGMA table_info({cls._table})")
+        if not row:
+            return False
+        columns = [col[1] for col in cls._db.fetchall(
+            f"PRAGMA table_info({cls._table})")]
+        return column_name in columns
+    
+    @classmethod
+    def rename_column(cls, old_column: str, new_column: str):
+        cls._db.execute(
+            f"ALTER TABLE {cls._table} RENAME COLUMN {old_column} TO {new_column}", commit=True)
+        
+    @classmethod
+    def drop_column(cls, column_name: str):
+        cls._db.execute(
+            f"ALTER TABLE {cls._table} DROP COLUMN {column_name}", commit=True)
+        
+    @classmethod
+    def get_column_type(cls, column_name: str) -> Optional[str]:
+        row = cls._db.fetchone(
+            f"PRAGMA table_info({cls._table})")
+        if not row:
+            return None
+        columns = cls._db.fetchall(
+            f"PRAGMA table_info({cls._table})")
+        for col in columns:
+            if col[1] == column_name:
+                return col[2]  # Return the type
+        return None
+        
+    @classmethod
+    def alter_column_type(cls, old_column: str, new_column: str, new_type: str):
+        # Add new column
+        cls._db.execute(
+            f"ALTER TABLE {cls._table} ADD COLUMN {new_column} {new_type}", commit=True)
+        # Copy and cast data
+        cls._db.execute(
+            f"UPDATE {cls._table} SET {new_column} = CAST({old_column} AS {new_type})", commit=True)
+        cls._db.execute(
+            f"ALTER TABLE {cls._table} DROP COLUMN {old_column}", commit=True)
 
 
 class AIModelsTable(SQLiteDBTable):
@@ -333,10 +377,10 @@ class AIModelsTable(SQLiteDBTable):
 
 class GameLinesTable(SQLiteDBTable):
     _table = 'game_lines'
-    _fields = ['game_name', 'line_text', 'timestamp', 'screenshot_in_anki',
-               'audio_in_anki', 'screenshot_path', 'audio_path', 'replay_path', 'translation']
+    _fields = ['game_name', 'line_text', 'screenshot_in_anki',
+               'audio_in_anki', 'screenshot_path', 'audio_path', 'replay_path', 'translation', 'timestamp']
     _types = [str,  # Includes primary key type
-              str, str, str, str, str, str, str, str, str]
+              str, str, str, str, str, str, str, str, float]
     _pk = 'id'
     _auto_increment = False  # Use string IDs
 
@@ -355,7 +399,7 @@ class GameLinesTable(SQLiteDBTable):
         self.game_name = game_name
         self.line_text = line_text
         self.context = context
-        self.timestamp = timestamp if timestamp is not None else datetime.now().timestamp()
+        self.timestamp = float(timestamp) if timestamp is not None else datetime.now().timestamp()
         self.screenshot_in_anki = screenshot_in_anki if screenshot_in_anki is not None else ''
         self.audio_in_anki = audio_in_anki if audio_in_anki is not None else ''
         self.screenshot_path = screenshot_path if screenshot_path is not None else ''
@@ -416,9 +460,79 @@ class GameLinesTable(SQLiteDBTable):
             params,
             commit=True
         )
+        
+    @classmethod
+    def get_lines_filtered_by_timestamp(cls, start: Optional[float] = None, end: Optional[float] = None) -> List['GameLinesTable']:
+        """
+        Fetches all lines optionally filtered by start and end timestamps.
+        If start or end is None, that bound is ignored.
+        """
+        query = f"SELECT * FROM {cls._table}"
+        conditions = []
+        params = []
+
+        # Add timestamp conditions if provided
+        if start is not None:
+            conditions.append("timestamp >= ?")
+            params.append(start)
+        if end is not None:
+            conditions.append("timestamp <= ?")
+            params.append(end)
+
+        # Combine conditions into WHERE clause if any
+        if conditions:
+            query += " WHERE " + " AND ".join(conditions)
+
+        # Sort by timestamp ascending
+        query += " ORDER BY timestamp ASC"
+
+        # Execute the query
+        rows = cls._db.fetchall(query, tuple(params))
+        return [cls.from_row(row) for row in rows]
+        
+class StatsRollupTable(SQLiteDBTable):
+    _table = 'stats_rollup'
+    _fields = ['date', 'games_played', 'lines_mined', 'anki_cards_created', 'time_spent_mining']
+    _types = [int,  # Includes primary key type
+              str, int, int, int, float]
+    _pk = 'id'
+    _auto_increment = True  # Use auto-incrementing integer IDs
+    
+    def __init__(self, id: Optional[int] = None,
+                 date: Optional[str] = None,
+                 games_played: int = 0,
+                 lines_mined: int = 0,
+                 anki_cards_created: int = 0,
+                 time_spent_mining: float = 0.0):
+        self.id = id
+        self.date = date if date is not None else datetime.now().strftime("%Y-%m-%d")
+        self.games_played = games_played
+        self.lines_mined = lines_mined
+        self.anki_cards_created = anki_cards_created
+        self.time_spent_mining = time_spent_mining
 
+    @classmethod
+    def get_stats_for_date(cls, date: str) -> Optional['StatsRollupTable']:
+        row = cls._db.fetchone(
+            f"SELECT * FROM {cls._table} WHERE date=?", (date,))
+        return cls.from_row(row) if row else None
 
-def get_db_directory():
+    @classmethod
+    def update_stats(cls, date: str, games_played: int = 0, lines_mined: int = 0, anki_cards_created: int = 0, time_spent_mining: float = 0.0):
+        stats = cls.get_stats_for_date(date)
+        if not stats:
+            new_stats = cls(date=date, games_played=games_played,
+                            lines_mined=lines_mined, anki_cards_created=anki_cards_created, time_spent_mining=time_spent_mining)
+            new_stats.save()
+            return
+        stats.games_played += games_played
+        stats.lines_mined += lines_mined
+        stats.anki_cards_created += anki_cards_created
+        stats.time_spent_mining += time_spent_mining
+        stats.save()
+        
+# Ensure database directory exists and return path
+def get_db_directory(test=False, delete_test=False) -> str:
     if platform == 'win32':  # Windows
         appdata_dir = os.getenv('APPDATA')
     else:  # macOS and Linux
@@ -426,7 +540,11 @@ def get_db_directory():
     config_dir = os.path.join(appdata_dir, 'GameSentenceMiner')
     # Create the directory if it doesn't exist
     os.makedirs(config_dir, exist_ok=True)
-    return os.path.join(config_dir, 'gsm.db')
+    path = os.path.join(config_dir, 'gsm.db' if not test else 'gsm_test.db')
+    if test and delete_test:
+        if os.path.exists(path):
+            os.remove(path)
+    return path
 
 
 # Backup and compress the database on load, with today's date, up to 5 days ago (clean up old backups)
@@ -462,6 +580,8 @@ db_path = get_db_directory()
 if os.path.exists(db_path):
     backup_db(db_path)
 
+# db_path = get_db_directory(test=True, delete_test=False)
+
 gsm_db = SQLiteDB(db_path)
 
 for cls in [AIModelsTable, GameLinesTable]:
@@ -470,13 +590,47 @@ for cls in [AIModelsTable, GameLinesTable]:
     # cls.drop()
     # cls.set_db(gsm_db)  # --- IGNORE ---
     
+# GameLinesTable.drop_column('timestamp')
+    
+# if GameLinesTable.has_column('timestamp_old'):
+#     GameLinesTable.alter_column_type('timestamp_old', 'timestamp', 'TEXT')
+#     logger.info("Altered 'timestamp_old' column to 'timestamp' with TEXT type in GameLinesTable.")
+
+def check_and_run_migrations():
+    def migrate_timestamp():
+        if GameLinesTable.has_column('timestamp') and GameLinesTable.get_column_type('timestamp') != 'REAL':
+            logger.info("Migrating 'timestamp' column to REAL type in GameLinesTable.")
+            # Rename 'timestamp' to 'timestamp_old'
+            GameLinesTable.rename_column('timestamp', 'timestamp_old')
+            # Copy and cast data from old column to new column
+            GameLinesTable.alter_column_type('timestamp_old', 'timestamp', 'REAL')
+            logger.info("Migrated 'timestamp' column to REAL type in GameLinesTable.")
+            
+            
+    migrate_timestamp()
+        
+check_and_run_migrations()
+    
+# all_lines = GameLinesTable.all()
+
+
+# # Convert String timestamp to float timestamp
+# for line in all_lines:
+#     if isinstance(line.timestamp, str):
+#         try:
+#             line.timestamp = float(line.timestamp)
+#         except ValueError:
+#             # Handle invalid timestamp format
+#             line.timestamp = 0.0
+#     line.save()
+
 # import random
 # import uuid
 # from datetime import datetime
 # from GameSentenceMiner.util.text_log import GameLine
 # from GameSentenceMiner.util.db import GameLinesTable
 
-# List of common Japanese characters (kanji, hiragana, katakana)
+# # List of common Japanese characters (kanji, hiragana, katakana)
 # japanese_chars = (
 #     "あいうえおかきくけこさしすせそたちつてとなにぬねのはひふへほまみむめもやゆよらりるれろわをん"
 #     "アイウエオカキクケコサシスセソタチツテトナニヌネノハヒフヘホマミムメモヤユヨラリルレロワヲン"
@@ -513,6 +667,7 @@ for cls in [AIModelsTable, GameLinesTable]:
     
 #     if len(lines_batch) >= batch_size:
 #         GameLinesTable.add_lines(lines_batch)
+#         GameLinesTable2.add_lines(lines_batch)
 #         lines_batch = []
 #     if i % 1000 == 0:
 #         print(f"Inserted {i} lines...")
@@ -520,5 +675,18 @@ for cls in [AIModelsTable, GameLinesTable]:
 # # Insert any remaining lines
 # if lines_batch:
 #     GameLinesTable.add_lines(lines_batch)
+#     GameLinesTable2.add_lines(lines_batch)
+# for _ in range(10):  # Run multiple times to see consistent timing
+#     start_time = time.time()
+#     GameLinesTable.all()
+#     end_time = time.time()
+
+#     print(f"Time taken to query all lines from GameLinesTable: {end_time - start_time:.2f} seconds")
+
+#     start_time = time.time()
+#     GameLinesTable2.all()
+#     end_time = time.time()
+
+#     print(f"Time taken to query all lines from GameLinesTable2: {end_time - start_time:.2f} seconds")
 
-print("Done populating GameLinesDB with random Japanese text.")
+# print("Done populating GameLinesTable and GameLinesTable2 with random Japanese text.")

GameSentenceMiner/util/downloader/download_tools.py

@@ -71,6 +71,7 @@ def download_obs_if_needed():
     def get_windows_obs_url():
         machine = platform.machine().lower()
         if machine in ['arm64', 'aarch64']:
+            logger.info("Detected Windows on ARM64. Getting ARM64 version of OBS Studio.")
             return next(asset['browser_download_url'] for asset in latest_release['assets'] if
                         asset['name'].endswith('Windows-arm64.zip'))
         return next(asset['browser_download_url'] for asset in latest_release['assets'] if 
@@ -80,12 +81,12 @@ def download_obs_if_needed():
     with urllib.request.urlopen(latest_release_url) as response:
         latest_release = json.load(response)
         obs_url = {
-            "Windows": get_windows_obs_url(),
-            "Linux": next(asset['browser_download_url'] for asset in latest_release['assets'] if
-                          asset['name'].endswith('Ubuntu-24.04-x86_64.deb')),
-            "Darwin": next(asset['browser_download_url'] for asset in latest_release['assets'] if
-                           asset['name'].endswith('macOS-Intel.dmg'))
-        }.get(platform.system(), None)
+            "Windows": get_windows_obs_url,
+            # "Linux": lambda: next(asset['browser_download_url'] for asset in latest_release['assets'] if
+            #                       asset['name'].endswith('Ubuntu-24.04-x86_64.deb')),
+            # "Darwin": lambda: next(asset['browser_download_url'] for asset in latest_release['assets'] if
+            #                        asset['name'].endswith('macOS-Intel.dmg'))
+        }.get(platform.system(), lambda: None)()
 
     if obs_url is None:
         logger.error("Unsupported OS. Please install OBS manually.")
@@ -163,43 +164,75 @@ def download_ffmpeg_if_needed():
         logger.info("FFmpeg directory exists but executables are missing. Re-downloading FFmpeg...")
         shutil.rmtree(ffmpeg_dir)
 
-    ffmpeg_url = {
-        "Windows": "https://www.gyan.dev/ffmpeg/builds/ffmpeg-release-essentials.zip",
-        "Linux": "https://johnvansickle.com/ffmpeg/releases/ffmpeg-release-amd64-static.tar.xz",
-        "Darwin": "https://evermeet.cx/ffmpeg/ffmpeg.zip"
-    }.get(platform.system(), None)
+    system = platform.system()
+    ffmpeg_url = None
+    compressed_format = "zip"
+    if system == "Windows":
+        machine = platform.machine().lower()
+        if machine in ['arm64', 'aarch64']:
+            ffmpeg_url = "https://gsm.beangate.us/ffmpeg-8.0-essentials-shared-win-arm64.zip"
+            compressed_format = "zip"
+        else:
+            ffmpeg_url = "https://www.gyan.dev/ffmpeg/builds/ffmpeg-release-essentials.zip"
+            compressed_format = "zip"
+    # elif system == "Linux":
+    #     ffmpeg_url = "https://johnvansickle.com/ffmpeg/releases/ffmpeg-release-amd64-static.tar.xz"
+    # elif system == "Darwin":
+    #     ffmpeg_url = "https://evermeet.cx/ffmpeg/ffmpeg.zip"
 
     if ffmpeg_url is None:
-        logger.error("Unsupported OS. Please install FFmpeg manually.")
+        logger.error("Unsupported OS/architecture. Please install FFmpeg manually.")
         return
 
     download_dir = os.path.join(get_app_directory(), "downloads")
     os.makedirs(download_dir, exist_ok=True)
-    ffmpeg_archive = os.path.join(download_dir, "ffmpeg.zip")
+    ffmpeg_archive = os.path.join(download_dir, f"ffmpeg.{compressed_format}")
 
     logger.info(f"Downloading FFmpeg from {ffmpeg_url}...")
     urllib.request.urlretrieve(ffmpeg_url, ffmpeg_archive)
     logger.info(f"FFmpeg downloaded. Extracting to {ffmpeg_dir}...")
 
     os.makedirs(ffmpeg_dir, exist_ok=True)
-
-    with zipfile.ZipFile(ffmpeg_archive, 'r') as zip_ref:
-        for member in zip_ref.namelist():
-            filename = os.path.basename(member)
-            if filename:  # Skip directories
-                source = zip_ref.open(member)
-                target = open(os.path.join(ffmpeg_dir, filename), "wb")
-                with source, target:
-                    shutil.copyfileobj(source, target)
+    
+    # Extract 7z
+    # Extract archive
+    if ffmpeg_url.endswith('.7z'):
+        with py7zr.SevenZipFile(ffmpeg_archive, mode='r') as z:
+            z.extractall(ffmpeg_dir)
+    else:
+        with zipfile.ZipFile(ffmpeg_archive, 'r') as zip_ref:
+            zip_ref.extractall(ffmpeg_dir)
+    
+    # Flatten directory structure - move all files to root ffmpeg_dir
+    def flatten_directory(directory):
+        for root, dirs, files in os.walk(directory):
+            for file in files:
+                file_path = os.path.join(root, file)
+                if root != directory:  # Only move files from subdirectories
+                    target_path = os.path.join(directory, file)
+                    # Handle name conflicts by keeping the first occurrence
+                    if not os.path.exists(target_path):
+                        shutil.move(file_path, target_path)
+        # Remove empty subdirectories
+        for root, dirs, files in os.walk(directory, topdown=False):
+            for dir_name in dirs:
+                dir_path = os.path.join(root, dir_name)
+                try:
+                    os.rmdir(dir_path)
+                except OSError:
+                    pass  # Directory not empty
+    
+    flatten_directory(ffmpeg_dir)
                     
     # Copy ffmpeg.exe to the python folder
     if os.path.exists(ffmpeg_exe_path):
         shutil.copy2(ffmpeg_exe_path, ffmpeg_in_python)
         logger.info(f"Copied ffmpeg.exe to Python folder: {ffmpeg_in_python}")
     else:
-        logger.warning(f"ffmpeg.exe not found in {ffmpeg_dir}.")
+        logger.warning(f"ffmpeg.exe not found in {ffmpeg_dir}. Extraction might have failed.")
     logger.info(f"FFmpeg extracted to {ffmpeg_dir}.")
 
+
 def download_ocenaudio_if_needed():
     ocenaudio_dir = os.path.join(get_app_directory(), 'ocenaudio')
     ocenaudio_exe_path = os.path.join(ocenaudio_dir, 'ocenaudio.exe')
@@ -234,4 +267,4 @@ def main():
     download_ocenaudio_if_needed()
 
 if __name__ == "__main__":
-    main()
+    main()

GameSentenceMiner/util/ffmpeg.py

@@ -1,13 +1,16 @@
+import json
+import os
 import subprocess
+import sys
 import tempfile
 import time
 from pathlib import Path
 
 from GameSentenceMiner import obs
+from GameSentenceMiner.util.configuration import get_app_directory, is_windows, logger, get_config, \
+    get_temporary_directory, gsm_state, is_linux
 from GameSentenceMiner.util.gsm_utils import make_unique_file_name, get_file_modification_time
 from GameSentenceMiner.util import configuration
-from GameSentenceMiner.util.configuration import *
-from GameSentenceMiner.util.model import VADResult
 from GameSentenceMiner.util.text_log import initial_time
 
 

GameSentenceMiner/util/get_overlay_coords.py

@@ -283,6 +283,9 @@ class OverlayProcessor:
         if not self.lens:
             logger.error("OCR engines are not initialized. Cannot perform OCR for Overlay.")
             return []
+        
+        if get_config().overlay.scan_delay > 0:
+            await asyncio.sleep(get_config().overlay.scan_delay)
 
         # 1. Get screenshot
         full_screenshot, monitor_width, monitor_height = self._get_full_screenshot()

GameSentenceMiner/util/gsm_utils.py

@@ -267,57 +267,3 @@ os.makedirs(os.path.dirname(TEXT_REPLACEMENTS_FILE), exist_ok=True)
 #                 f.write(data)
 #     except Exception as e:
 #         logger.error(f"Failed to fetch JSON from {url}: {e}")
-
-
-# Remove GitHub replacements from local OCR replacements file, these replacements are not needed
-def remove_github_replacements_from_local_ocr():
-    github_url = "https://raw.githubusercontent.com/bpwhelan/GameSentenceMiner/main/electron-src/assets/ocr_replacements.json"
-
-    github_replacements = {}
-    try:
-        response = requests.get(github_url)
-        response.raise_for_status()
-        github_data = response.json()
-        github_replacements = github_data.get('args', {}).get('replacements', {})
-        logger.debug(f"Successfully fetched {len(github_replacements)} replacements from GitHub.")
-    except requests.exceptions.RequestException as e:
-        logger.debug(f"Failed to fetch GitHub replacements from {github_url}: {e}")
-        return
-    except json.JSONDecodeError as e:
-        logger.debug(f"Error decoding JSON from GitHub response: {e}")
-        return
-
-    if not os.path.exists(OCR_REPLACEMENTS_FILE):
-        logger.debug(f"Local file {OCR_REPLACEMENTS_FILE} does not exist. No replacements to remove.")
-        return
-
-    try:
-        with open(OCR_REPLACEMENTS_FILE, 'r', encoding='utf-8') as f:
-            local_ocr_data = json.load(f)
-
-        local_replacements = local_ocr_data.get('args', {}).get('replacements', {})
-        original_count = len(local_replacements)
-        logger.debug(f"Loaded {original_count} replacements from local file.")
-
-        removed_count = 0
-        for key_to_remove in github_replacements.keys():
-            if key_to_remove in local_replacements:
-                del local_replacements[key_to_remove]
-                removed_count += 1
-
-        if removed_count > 0:
-            local_ocr_data['args']['replacements'] = local_replacements
-            with open(OCR_REPLACEMENTS_FILE, 'w', encoding='utf-8') as f:
-                json.dump(local_ocr_data, f, ensure_ascii=False, indent=4)
-            logger.debug(f"Successfully removed {removed_count} replacements from {OCR_REPLACEMENTS_FILE}.")
-            logger.debug(f"Remaining replacements in local file: {len(local_replacements)}")
-        else:
-            logger.debug("No matching replacements from GitHub found in your local file to remove.")
-
-    except json.JSONDecodeError as e:
-        logger.debug(f"Error decoding JSON from {OCR_REPLACEMENTS_FILE}: {e}. Please ensure it's valid JSON.")
-    except Exception as e:
-        logger.debug(f"An unexpected error occurred while processing {OCR_REPLACEMENTS_FILE}: {e}")
-
-
-remove_github_replacements_from_local_ocr()

GameSentenceMiner/vad.py

@@ -1,11 +1,14 @@
+import json
+import logging
+import os
+import shutil
 import tempfile
 import time
 import warnings
-import requests
 from abc import abstractmethod, ABC
 
 from GameSentenceMiner.util import configuration, ffmpeg
-from GameSentenceMiner.util.configuration import *
+from GameSentenceMiner.util.configuration import get_config, get_temporary_directory, logger, SILERO, WHISPER
 from GameSentenceMiner.util.ffmpeg import get_audio_length
 from GameSentenceMiner.util.gsm_utils import make_unique_file_name, run_new_thread
 from GameSentenceMiner.util.model import VADResult

GameSentenceMiner/web/database_api.py

@@ -795,9 +795,20 @@ def register_database_api_routes(app):
         punctionation_regex = regex.compile(r'[\p{P}\p{S}\p{Z}]')
         # Get optional year filter parameter
         filter_year = request.args.get('year', None)
+
+        # Get Start and End time as unix timestamp
+        start_timestamp = request.args.get('start', None)
+        end_timestamp = request.args.get('end', None)
         
+        # Convert timestamps to float if provided
+        start_timestamp = float(start_timestamp) if start_timestamp else None
+        end_timestamp = float(end_timestamp) if end_timestamp else None
+
+        # # 1. Fetch all lines and sort them chronologically
+        # all_lines = sorted(GameLinesTable.all(), key=lambda line: line.timestamp)
+
         # 1. Fetch all lines and sort them chronologically
-        all_lines = sorted(GameLinesTable.all(), key=lambda line: line.timestamp)
+        all_lines = GameLinesTable.get_lines_filtered_by_timestamp(start=start_timestamp, end=end_timestamp)
         
         if not all_lines:
             return jsonify({"labels": [], "datasets": []})

GameSentenceMiner/web/gsm_websocket.py

@@ -78,7 +78,7 @@ class WebsocketServerThread(threading.Thread):
             while True:
                 try:
                     self.server = start_server = websockets.serve(self.server_handler,
-                                                                  "0.0.0.0",
+                                                                  get_config().advanced.localhost_bind_address,
                                                                   self.get_ws_port_func(),
                                                                   max_size=1000000000)
                     async with start_server:

GameSentenceMiner/web/static/css/shared.css

@@ -929,4 +929,24 @@ h1 {
     .games-table td:nth-child(6) {
         display: none;
     }
+}
+
+input[type="date"].dashboard-date-input {
+    font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif;
+    transition: all 0.2s ease; /* smooth animation */
+    border-radius: 8px;
+    border: 1px solid var(--border-color);
+    padding: 8px 12px;
+    font-size: 14px;
+    font-weight: 500;
+    color: var(--text-primary);
+    background: var(--bg-tertiary);
+    cursor: pointer;
+}
+
+input[type="date"].dashboard-date-input:focus,
+input[type="date"].dashboard-date-input:hover {
+    border-color: var(--accent-color);
+    box-shadow: 0 0 0 2px rgba(0, 123, 255, 0.15);
+    outline: none;
 }