GameSentenceMiner 2.15.9__py3-none-any.whl → 2.15.11__py3-none-any.whl

GameSentenceMiner/web/texthooking_page.py

@@ -1,201 +1,52 @@
 import asyncio
-from ctypes.util import test
 import datetime
 import json
 import os
-import queue
-import sqlite3
 import threading
-from dataclasses import dataclass
 
 import flask
-import websockets
+import webbrowser
 
 from GameSentenceMiner.ai.ai_prompting import get_ai_prompt_result
 from GameSentenceMiner.obs import get_current_game
 from GameSentenceMiner.util.gsm_utils import TEXT_REPLACEMENTS_FILE
-from GameSentenceMiner.util.text_log import GameLine, get_line_by_id, initial_time, get_all_lines
-from flask import request, jsonify, send_from_directory
-import webbrowser
+from GameSentenceMiner.util.text_log import get_line_by_id, get_all_lines
+from flask import render_template, request, jsonify, send_from_directory
 from GameSentenceMiner import obs
-from GameSentenceMiner.util.configuration import logger, get_config, DB_PATH, gsm_state, gsm_status
+from GameSentenceMiner.util.configuration import logger, get_config, gsm_state, gsm_status
 from GameSentenceMiner.web.service import handle_texthooker_button
 
+# Import from new modules
+from GameSentenceMiner.web.events import (
+    EventItem, EventManager, EventProcessor, event_manager, event_queue, event_processor
+)
+from GameSentenceMiner.web.stats import (
+    is_kanji, interpolate_color, get_gradient_color, calculate_kanji_frequency,
+    calculate_heatmap_data, calculate_total_chars_per_game, calculate_reading_time_per_game,
+    calculate_reading_speed_per_game, generate_game_colors, format_large_number,
+    calculate_actual_reading_time, calculate_daily_reading_time, calculate_time_based_streak,
+    format_time_human_readable, calculate_current_game_stats, calculate_all_games_stats
+)
+from GameSentenceMiner.web.websockets import (
+    WebsocketServerThread, websocket_queue, paused, websocket_server_thread,
+    plaintext_websocket_server_thread, overlay_server_thread, websocket_server_threads,
+    handle_exit_signal
+)
+from GameSentenceMiner.web.database_api import register_database_api_routes
+
+# Global configuration
 port = get_config().general.texthooker_port
 url = f"http://localhost:{port}"
 websocket_port = 55001
 
-
-@dataclass
-class EventItem:
-    line: 'GameLine'
-    id: str
-    text: str
-    time: datetime.datetime
-    checked: bool = False
-    history: bool = False
-
-    def to_dict(self):
-        return {
-            'id': self.id,
-            'text': self.text,
-            'time': self.time,
-            'checked': self.checked,
-            'history': self.history,
-        }
-
-    def to_serializable(self):
-        return {
-            'id': self.id,
-            'text': self.text,
-            'time': self.time.isoformat(),
-            'checked': self.checked,
-            'history': self.history,
-        }
-
-
-class EventManager:
-    events: list[EventItem]
-    events_dict: dict[str, EventItem] = {}
-
-    def __init__(self):
-        self.events = []
-        self.ids = []
-        self.events_dict = {}
-        self._connect()
-        self._create_table()
-        self._load_events_from_db()
-        # self.close_connection()
-
-    def _connect(self):
-        self.conn = sqlite3.connect(DB_PATH)
-        self.cursor = self.conn.cursor()
-
-    def _create_table(self):
-        self.cursor.execute("""
-            CREATE TABLE IF NOT EXISTS events (
-                event_id TEXT PRIMARY KEY,
-                line_id TEXT,
-                text TEXT,
-                time TEXT
-            )
-        """)
-        self.conn.commit()
-
-    def _load_events_from_db(self):
-        self.cursor.execute("SELECT * FROM events")
-        rows = self.cursor.fetchall()
-        for row in rows:
-            event_id, line_id, text, timestamp = row
-            timestamp = datetime.datetime.fromisoformat(timestamp)
-            line = GameLine(line_id, text, timestamp, None, None, 0)
-            event = EventItem(line, event_id, text, timestamp,
-                              False, timestamp < initial_time)
-            self.events.append(event)
-            self.ids.append(event_id)
-            self.events_dict[event_id] = event
-
-    def __iter__(self):
-        return iter(self.events)
-
-    def replace_events(self, new_events: list[EventItem]):
-        self.events = new_events
-
-    def add_gameline(self, line: GameLine):
-        new_event = EventItem(line, line.id, line.text,
-                              line.time, False, False)
-        self.events_dict[line.id] = new_event
-        self.ids.append(line.id)
-        self.events.append(new_event)
-        # self.store_to_db(new_event)
-        # event_queue.put(new_event)
-        return new_event
-
-    def reset_checked_lines(self):
-        for event in self.events:
-            event.checked = False
-
-    def get_events(self):
-        return self.events
-
-    def add_event(self, event):
-        self.events.append(event)
-        self.ids.append(event.id)
-        event_queue.put(event)
-
-    def get(self, event_id):
-        return self.events_dict.get(event_id)
-
-    def get_ids(self):
-        return self.ids
-
-    def close_connection(self):
-        if self.conn:
-            self.conn.close()
-
-    def clear_history(self):
-        self.cursor.execute("DELETE FROM events WHERE time < ?",
-                            (initial_time.isoformat(),))
-        logger.info(f"Cleared history before {initial_time.isoformat()}")
-        self.conn.commit()
-        # Clear the in-memory events as well
-        event_manager.events = [
-            event for event in event_manager if not event.history]
-        event_manager.events_dict = {
-            event.id: event for event in event_manager.events}
-
-
-class EventProcessor(threading.Thread):
-    def __init__(self, event_queue, db_path):
-        super().__init__()
-        self.event_queue = event_queue
-        self.db_path = db_path
-        self.conn = None
-        self.cursor = None
-        self.daemon = True
-
-    def _connect(self):
-        self.conn = sqlite3.connect(self.db_path)
-        self.cursor = self.conn.cursor()
-
-    def run(self):
-        self._connect()
-        while True:
-            try:
-                event = self.event_queue.get()
-                if event is None:  # Exit signal
-                    break
-                self._store_to_db(event)
-            except Exception as e:
-                logger.error(f"Error processing event: {e}")
-        self._close_connection()
-
-    def _store_to_db(self, event):
-        self.cursor.execute("""
-            INSERT INTO events (event_id, line_id, text, time)
-            VALUES (?, ?, ?, ?)
-        """, (event.id, event.line.id, event.text, event.time.isoformat()))
-        self.conn.commit()
-
-    def _close_connection(self):
-        if self.conn:
-            self.conn.close()
-
-
-event_manager = EventManager()
-event_queue = queue.Queue()
-
-# Initialize the EventProcessor with the queue and event manager
-event_processor = EventProcessor(event_queue, DB_PATH)
-event_processor.start()
-
 server_start_time = datetime.datetime.now().timestamp()
 
 app = flask.Flask(__name__)
 
-# Load data from the JSON file
-
+# Register database API routes
+register_database_api_routes(app)
 
+# Load data from the JSON file
 def load_data_from_file():
     if os.path.exists(TEXT_REPLACEMENTS_FILE):
         with open(TEXT_REPLACEMENTS_FILE, 'r', encoding='utf-8') as file:
@@ -203,8 +54,6 @@ def load_data_from_file():
     return {"enabled": True, "args": {"replacements": {}}}
 
 # Save data to the JSON file
-
-
 def save_data_to_file(data):
     with open(TEXT_REPLACEMENTS_FILE, 'w', encoding='utf-8') as file:
         json.dump(data, file, indent=4, ensure_ascii=False)
@@ -262,7 +111,19 @@ def texthooker():
 
 @app.route('/textreplacements')
 def textreplacements():
-    return flask.render_template('text_replacements.html')
+    # Serve the text replacements data as JSON for compatibility
+    try:
+        if not os.path.exists(TEXT_REPLACEMENTS_FILE):
+            return jsonify({"error": "Text replacements file not found."}), 404
+        with open(TEXT_REPLACEMENTS_FILE, "r", encoding="utf-8") as f:
+            data = json.load(f)
+        return jsonify(data)
+    except Exception as e:
+        return jsonify({"error": f"Failed to load text replacements: {str(e)}"}), 500
+
+@app.route('/database')
+def database():
+    return flask.render_template('database.html')
 
 
 @app.route('/data', methods=['GET'])
@@ -283,7 +144,7 @@ def clear_history():
     return jsonify({'message': 'History cleared successfully'}), 200
 
 
-async def add_event_to_texthooker(line: GameLine):
+async def add_event_to_texthooker(line):
     new_event = event_manager.add_gameline(line)
     await websocket_server_thread.send_text({
         'event': 'text_received',
@@ -389,32 +250,74 @@ Translate the following lines of game dialogue into natural-sounding, context-aw
 def get_status():
     return jsonify(gsm_status.to_dict()), 200
 
+@app.template_filter('datetimeformat')
+def datetimeformat(value, format='%Y-%m-%d %H:%M:%S'):
+    """Formats a timestamp into a human-readable string."""
+    if value is None:
+        return ""
+    return datetime.datetime.fromtimestamp(float(value)).strftime(format)
+
+
+@app.route('/stats')
+def stats():
+    """Renders the stats page."""
+    return render_template('stats.html')
+
+@app.route('/api/anki_stats')
+def api_anki_stats():
+    """
+    API endpoint to provide Anki vs GSM kanji stats for the frontend.
+    Returns:
+        {
+            "missing_kanji": [ { "kanji": "漢", "frequency": 42 }, ... ],
+            "anki_kanji_count": 123,
+            "gsm_kanji_count": 456,
+            "coverage_percent": 27.0
+        }
+    """
+    from GameSentenceMiner.anki import get_all_anki_first_field_kanji
+    from GameSentenceMiner.web.stats import calculate_kanji_frequency, is_kanji
+    from GameSentenceMiner.util.db import GameLinesTable
+
+    # Get all GSM lines and calculate kanji frequency
+    all_lines = GameLinesTable.all()
+    gsm_kanji_stats = calculate_kanji_frequency(all_lines)
+    gsm_kanji_list = gsm_kanji_stats.get("kanji_data", [])
+    gsm_kanji_set = set([k["kanji"] for k in gsm_kanji_list])
+
+    # Get all kanji in Anki (first field only)
+    anki_kanji_set = get_all_anki_first_field_kanji()
+
+    # Find missing kanji (in GSM but not in Anki)
+    missing_kanji = [
+        {"kanji": k["kanji"], "frequency": k["frequency"]}
+        for k in gsm_kanji_list if k["kanji"] not in anki_kanji_set
+    ]
+
+    # Sort missing kanji by frequency descending
+    missing_kanji.sort(key=lambda x: x["frequency"], reverse=True)
+
+    # Coverage stats
+    anki_kanji_count = len(anki_kanji_set)
+    gsm_kanji_count = len(gsm_kanji_set)
+    coverage_percent = (anki_kanji_count / gsm_kanji_count * 100) if gsm_kanji_count else 0.0
+
+    return jsonify({
+        "missing_kanji": missing_kanji,
+        "anki_kanji_count": anki_kanji_count,
+        "gsm_kanji_count": gsm_kanji_count,
+        "coverage_percent": round(coverage_percent, 1)
+    })
+
+@app.route('/search')
+def search():
+    """Renders the search page."""
+    return render_template('search.html')
 
-# async def main():
-#     async with websockets.serve(websocket_handler, "localhost", 8765): # Choose a port for WebSocket
-#         print("WebSocket server started on ws://localhost:8765/ws (adjust as needed)")
-#         await asyncio.Future()  # Keep the server running
-
-# @app.route('/store-events', methods=['POST'])
-# def store_events():
-#     data = request.get_json()
-#     events_data = data.get('events', [])
-#
-#     if not isinstance(events_data, list):
-#         return jsonify({'error': 'Invalid data format. Expected an array of events.'}), 400
-#
-#     for event_data in events_data:
-#         if not all(k in event_data for k in ('id', 'text', 'time', 'checked')):
-#              return jsonify({'error': 'Invalid event structure. Missing keys.'}), 400
-#         if not (isinstance(event_data['id'], (int, float)) and
-#                 isinstance(event_data['text'], str) and
-#                 isinstance(event_data['time'], str) and
-#                 isinstance(event_data['checked'], bool)):
-#             return jsonify({'error': 'Invalid event structure. Incorrect data types.'}), 400
-#
-#     event_manager.replace_events([EventItem(item['id'], item['text'], item['time'], item.get(['timestamp'], 0), item['checked']) for item in data])
-#
-#     return jsonify({'message': 'Events successfully stored on server.', 'receivedEvents': data}), 200
+@app.route('/anki_stats')
+def anki_stats():
+    """Renders the Anki statistics page."""
+    return render_template('anki_stats.html')
 
 
 def get_selected_lines():
@@ -452,117 +355,6 @@ def start_web_server():
     app.run(host='0.0.0.0', port=port, debug=False)
 
 
-websocket_queue = queue.Queue()
-paused = False
-
-
-class WebsocketServerThread(threading.Thread):
-    def __init__(self, read, get_ws_port_func):
-        super().__init__(daemon=True)
-        self._loop = None
-        self.read = read
-        self.clients = set()
-        self._event = threading.Event()
-        self.get_ws_port_func = get_ws_port_func
-        self.backedup_text = []
-
-    @property
-    def loop(self):
-        self._event.wait()
-        return self._loop
-
-    async def send_text_coroutine(self, message):
-        if not self.clients:
-            self.backedup_text.append(message)
-            return
-        for client in self.clients:
-            await client.send(message)
-
-    async def server_handler(self, websocket):
-        self.clients.add(websocket)
-        try:
-            if self.backedup_text:
-                for message in self.backedup_text:
-                    await websocket.send(message)
-                self.backedup_text.clear()
-            async for message in websocket:
-                if self.read and not paused:
-                    websocket_queue.put(message)
-                    try:
-                        await websocket.send('True')
-                    except websockets.exceptions.ConnectionClosedOK:
-                        pass
-                else:
-                    try:
-                        await websocket.send('False')
-                    except websockets.exceptions.ConnectionClosedOK:
-                        pass
-        except websockets.exceptions.ConnectionClosedError:
-            pass
-        finally:
-            self.clients.remove(websocket)
-
-    async def send_text(self, text):
-        if text:
-            if isinstance(text, dict) or isinstance(text, list):
-                text = json.dumps(text)
-            return asyncio.run_coroutine_threadsafe(
-                self.send_text_coroutine(text), self.loop)
-
-    def has_clients(self):
-        return len(self.clients) > 0
-
-    def stop_server(self):
-        self.loop.call_soon_threadsafe(self._stop_event.set)
-
-    def run(self):
-        async def main():
-            self._loop = asyncio.get_running_loop()
-            self._stop_event = stop_event = asyncio.Event()
-            self._event.set()
-            while True:
-                try:
-                    self.server = start_server = websockets.serve(self.server_handler,
-                                                                  "0.0.0.0",
-                                                                  self.get_ws_port_func(),
-                                                                  max_size=1000000000)
-                    async with start_server:
-                        await stop_event.wait()
-                    return
-                except Exception as e:
-                    logger.warning(
-                        f"WebSocket server encountered an error: {e}. Retrying...")
-                    await asyncio.sleep(1)
-
-        asyncio.run(main())
-
-
-def handle_exit_signal(loop):
-    logger.info("Received exit signal. Shutting down...")
-    for task in asyncio.all_tasks(loop):
-        task.cancel()
-
-
-websocket_server_thread = WebsocketServerThread(read=True, get_ws_port_func=lambda: get_config(
-).get_field_value('advanced', 'texthooker_communication_websocket_port'))
-websocket_server_thread.start()
-
-if get_config().advanced.plaintext_websocket_port:
-    plaintext_websocket_server_thread = WebsocketServerThread(
-        read=False, get_ws_port_func=lambda: get_config().get_field_value('advanced', 'plaintext_websocket_port'))
-    plaintext_websocket_server_thread.start()
-
-overlay_server_thread = WebsocketServerThread(
-    read=False, get_ws_port_func=lambda: get_config().get_field_value('overlay', 'websocket_port'))
-overlay_server_thread.start()
-
-websocket_server_threads = [
-    websocket_server_thread,
-    plaintext_websocket_server_thread,
-    overlay_server_thread
-]
-
-
 async def texthooker_page_coro():
     global websocket_server_thread, plaintext_websocket_server_thread, overlay_server_thread
     # Run the WebSocket server in the asyncio event loop
@@ -581,4 +373,4 @@ def run_text_hooker_page():
 
 
 if __name__ == '__main__':
-    asyncio.run(texthooker_page_coro())
+    asyncio.run(texthooker_page_coro())

GameSentenceMiner/web/websockets.py

@@ -0,0 +1,120 @@
+import asyncio
+import json
+import queue
+import threading
+import websockets
+
+from GameSentenceMiner.util.configuration import logger, get_config
+
+
+websocket_queue = queue.Queue()
+paused = False
+
+
+class WebsocketServerThread(threading.Thread):
+    def __init__(self, read, get_ws_port_func):
+        super().__init__(daemon=True)
+        self._loop = None
+        self.read = read
+        self.clients = set()
+        self._event = threading.Event()
+        self.get_ws_port_func = get_ws_port_func
+        self.backedup_text = []
+
+    @property
+    def loop(self):
+        self._event.wait()
+        return self._loop
+
+    async def send_text_coroutine(self, message):
+        if not self.clients:
+            self.backedup_text.append(message)
+            return
+        for client in self.clients:
+            await client.send(message)
+
+    async def server_handler(self, websocket):
+        self.clients.add(websocket)
+        try:
+            if self.backedup_text:
+                for message in self.backedup_text:
+                    await websocket.send(message)
+                self.backedup_text.clear()
+            async for message in websocket:
+                if self.read and not paused:
+                    websocket_queue.put(message)
+                    try:
+                        await websocket.send('True')
+                    except websockets.exceptions.ConnectionClosedOK:
+                        pass
+                else:
+                    try:
+                        await websocket.send('False')
+                    except websockets.exceptions.ConnectionClosedOK:
+                        pass
+        except websockets.exceptions.ConnectionClosedError:
+            pass
+        finally:
+            self.clients.remove(websocket)
+
+    async def send_text(self, text):
+        if text:
+            if isinstance(text, dict) or isinstance(text, list):
+                text = json.dumps(text)
+            return asyncio.run_coroutine_threadsafe(
+                self.send_text_coroutine(text), self.loop)
+
+    def has_clients(self):
+        return len(self.clients) > 0
+
+    def stop_server(self):
+        self.loop.call_soon_threadsafe(self._stop_event.set)
+
+    def run(self):
+        async def main():
+            self._loop = asyncio.get_running_loop()
+            self._stop_event = stop_event = asyncio.Event()
+            self._event.set()
+            while True:
+                try:
+                    self.server = start_server = websockets.serve(self.server_handler,
+                                                                  "0.0.0.0",
+                                                                  self.get_ws_port_func(),
+                                                                  max_size=1000000000)
+                    async with start_server:
+                        await stop_event.wait()
+                    return
+                except Exception as e:
+                    logger.warning(
+                        f"WebSocket server encountered an error: {e}. Retrying...")
+                    await asyncio.sleep(1)
+
+        asyncio.run(main())
+
+
+def handle_exit_signal(loop):
+    logger.info("Received exit signal. Shutting down...")
+    for task in asyncio.all_tasks(loop):
+        task.cancel()
+
+
+# Initialize WebSocket server threads
+websocket_server_thread = WebsocketServerThread(read=True, get_ws_port_func=lambda: get_config(
+).get_field_value('advanced', 'texthooker_communication_websocket_port'))
+websocket_server_thread.start()
+
+plaintext_websocket_server_thread = None
+if get_config().advanced.plaintext_websocket_port:
+    plaintext_websocket_server_thread = WebsocketServerThread(
+        read=False, get_ws_port_func=lambda: get_config().get_field_value('advanced', 'plaintext_websocket_port'))
+    plaintext_websocket_server_thread.start()
+
+overlay_server_thread = WebsocketServerThread(
+    read=False, get_ws_port_func=lambda: get_config().get_field_value('overlay', 'websocket_port'))
+overlay_server_thread.start()
+
+websocket_server_threads = [
+    websocket_server_thread,
+    plaintext_websocket_server_thread,
+    overlay_server_thread
+]

{gamesentenceminer-2.15.9.dist-info → gamesentenceminer-2.15.11.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: GameSentenceMiner
-Version: 2.15.9
+Version: 2.15.11
 Summary: A tool for mining sentences from games. Update: Overlay?
 Author-email: Beangate <bpwhelan95@gmail.com>
 License: MIT License