GameSentenceMiner 2.19.16__py3-none-any.whl → 2.20.0__py3-none-any.whl

GameSentenceMiner/web/texthooking_page.py

@@ -6,6 +6,7 @@ import threading
 
 import flask
 import webbrowser
+from flask import make_response
 
 from GameSentenceMiner.ai.ai_prompting import get_ai_prompt_result
 from GameSentenceMiner.obs import get_current_game
@@ -13,26 +14,47 @@ from GameSentenceMiner.util.gsm_utils import TEXT_REPLACEMENTS_FILE
 from GameSentenceMiner.util.text_log import get_line_by_id, get_all_lines
 from flask import render_template, request, jsonify, send_from_directory
 from GameSentenceMiner import obs
-from GameSentenceMiner.util.configuration import logger, get_config, gsm_state, gsm_status
+from GameSentenceMiner.util.configuration import (
+    logger,
+    get_config,
+    gsm_state,
+    gsm_status,
+)
 from GameSentenceMiner.web.service import handle_texthooker_button
 
 # Import from new modules
-from GameSentenceMiner.web.events import (
-    EventManager, event_manager
-)
+from GameSentenceMiner.web.events import EventManager, event_manager
 from GameSentenceMiner.web.stats import (
-    is_kanji, interpolate_color, get_gradient_color, calculate_kanji_frequency,
-    calculate_heatmap_data, calculate_total_chars_per_game, calculate_reading_time_per_game,
-    calculate_reading_speed_per_game, generate_game_colors, format_large_number,
-    calculate_actual_reading_time, calculate_daily_reading_time, calculate_time_based_streak,
-    format_time_human_readable, calculate_current_game_stats, calculate_all_games_stats
+    is_kanji,
+    interpolate_color,
+    get_gradient_color,
+    calculate_kanji_frequency,
+    calculate_heatmap_data,
+    calculate_total_chars_per_game,
+    calculate_reading_time_per_game,
+    calculate_reading_speed_per_game,
+    generate_game_colors,
+    format_large_number,
+    calculate_actual_reading_time,
+    calculate_daily_reading_time,
+    calculate_time_based_streak,
+    format_time_human_readable,
+    calculate_current_game_stats,
+    calculate_all_games_stats,
 )
 from GameSentenceMiner.web.gsm_websocket import (
-    WebsocketServerThread, websocket_queue, paused, websocket_server_thread,
-    plaintext_websocket_server_thread, overlay_server_thread, websocket_server_threads,
-    handle_exit_signal
+    WebsocketServerThread,
+    websocket_queue,
+    paused,
+    websocket_server_thread,
+    plaintext_websocket_server_thread,
+    overlay_server_thread,
+    websocket_server_threads,
+    handle_exit_signal,
 )
 from GameSentenceMiner.web.database_api import register_database_api_routes
+from GameSentenceMiner.web.jiten_database_api import register_jiten_database_api_routes
+from GameSentenceMiner.web.stats_api import register_stats_api_routes
 
 # Global configuration
 port = get_config().general.texthooker_port
@@ -43,27 +65,102 @@ server_start_time = datetime.datetime.now().timestamp()
 
 app = flask.Flask(__name__)
 
+# Configure Flask-Compress for Brotli compression
+try:
+    from flask_compress import Compress
+
+    # Configure compression settings
+    app.config["COMPRESS_MIMETYPES"] = [
+        "text/html",
+        "text/css",
+        "text/xml",
+        "text/plain",
+        "application/json",
+        "application/javascript",
+        "application/x-javascript",
+        "text/javascript",
+    ]
+    app.config["COMPRESS_LEVEL"] = 6  # Balance between speed and compression ratio
+    app.config["COMPRESS_MIN_SIZE"] = 500  # Only compress files larger than 500 bytes
+    app.config["COMPRESS_ALGORITHM"] = [
+        "br",
+        "gzip",
+        "deflate",
+    ]  # Prefer Brotli, fallback to gzip
+
+    Compress(app)
+    logger.info("Flask compression enabled with Brotli support")
+except ImportError:
+    logger.warning(
+        "flask-compress not installed. Run 'pip install flask-compress' for better performance."
+    )
+
+
+# Add cache control headers for static files
+@app.after_request
+def add_cache_headers(response):
+    """Add cache control headers to static assets for better performance."""
+    # Only add cache headers for static files (CSS, JS, images, fonts)
+    if request.path.startswith("/static/"):
+        # Check file extension
+        if any(request.path.endswith(ext) for ext in [".css", ".js"]):
+            # No cache for CSS/JS - always fetch fresh content
+            response.cache_control.no_cache = True
+            response.cache_control.no_store = True
+            response.cache_control.must_revalidate = True
+            response.headers["Cache-Control"] = (
+                "no-cache, no-store, must-revalidate"
+            )
+            response.headers["Pragma"] = "no-cache"
+            response.headers["Expires"] = "0"
+        elif any(
+            request.path.endswith(ext)
+            for ext in [
+                ".jpg",
+                ".jpeg",
+                ".png",
+                ".gif",
+                ".svg",
+                ".woff",
+                ".woff2",
+                ".ttf",
+                ".eot",
+                ".ico",
+            ]
+        ):
+            # Cache images and fonts for longer (they rarely change)
+            response.cache_control.max_age = 2592000  # 30 days
+            response.cache_control.public = True
+            response.headers["Cache-Control"] = "public, max-age=2592000, immutable"
+    return response
+
+
 # Register database API routes
 register_database_api_routes(app)
+register_jiten_database_api_routes(app)
+register_stats_api_routes(app)
 
 # Register Anki API routes
 from GameSentenceMiner.web.anki_api_endpoints import register_anki_api_endpoints
+
 register_anki_api_endpoints(app)
 
+
 # Load data from the JSON file
 def load_data_from_file():
     if os.path.exists(TEXT_REPLACEMENTS_FILE):
-        with open(TEXT_REPLACEMENTS_FILE, 'r', encoding='utf-8') as file:
+        with open(TEXT_REPLACEMENTS_FILE, "r", encoding="utf-8") as file:
             return json.load(file)
     return {"enabled": True, "args": {"replacements": {}}}
 
+
 # Save data to the JSON file
 def save_data_to_file(data):
-    with open(TEXT_REPLACEMENTS_FILE, 'w', encoding='utf-8') as file:
+    with open(TEXT_REPLACEMENTS_FILE, "w", encoding="utf-8") as file:
         json.dump(data, file, indent=4, ensure_ascii=False)
 
 
-@app.route('/load-data', methods=['GET'])
+@app.route("/load-data", methods=["GET"])
 def load_data():
     try:
         data = load_data_from_file()
@@ -72,7 +169,7 @@ def load_data():
         return jsonify({"error": f"Failed to load data: {str(e)}"}), 500
 
 
-@app.route('/save-data', methods=['POST'])
+@app.route("/save-data", methods=["POST"])
 def save_data():
     try:
         data = request.get_json()
@@ -87,33 +184,36 @@ def save_data():
 
 
 def inject_server_start_time(html_content, timestamp):
-    placeholder = '<script>'
-    replacement = f'<script>const serverStartTime = {timestamp};'
+    placeholder = "<script>"
+    replacement = f"<script>const serverStartTime = {timestamp};"
     return html_content.replace(placeholder, replacement)
 
 
-@app.route('/favicon.ico')
+@app.route("/favicon.ico")
 def favicon():
-    return send_from_directory(os.path.join(app.root_path, 'static'),
-                               'favicon.ico', mimetype='image/vnd.microsoft.icon')
+    return send_from_directory(
+        os.path.join(app.root_path, "static"),
+        "favicon.ico",
+        mimetype="image/vnd.microsoft.icon",
+    )
 
 
-@app.route('/<path:filename>')
+@app.route("/<path:filename>")
 def serve_static(filename):
-    return send_from_directory('pages', filename)
+    return send_from_directory("pages", filename)
 
 
-@app.route('/')
+@app.route("/")
 def index():
-    return send_from_directory('templates', 'index.html')
+    return send_from_directory("templates", "index.html")
 
 
-@app.route('/texthooker')
+@app.route("/texthooker")
 def texthooker():
-    return send_from_directory('templates', 'index.html')
+    return send_from_directory("templates", "index.html")
 
 
-@app.route('/textreplacements')
+@app.route("/textreplacements")
 def textreplacements():
     # Serve the text replacements data as JSON for compatibility
     try:
@@ -125,48 +225,59 @@ def textreplacements():
     except Exception as e:
         return jsonify({"error": f"Failed to load text replacements: {str(e)}"}), 500
 
-@app.route('/database')
+
+@app.route("/database")
 def database():
-    return flask.render_template('database.html')
+    return flask.render_template("database.html")
 
 
-@app.route('/data', methods=['GET'])
+@app.route("/data", methods=["GET"])
 def get_data():
     return jsonify([event.to_dict() for event in event_manager])
 
 
-@app.route('/get_ids', methods=['GET'])
+@app.route("/get_ids", methods=["GET"])
 def get_ids():
     asyncio.run(check_for_lines_outside_replay_buffer())
-    return jsonify({
-        "ids": list(event_manager.get_ids()),
-        "timed_out_ids": list(event_manager.timed_out_ids)
-    })
+    return jsonify(
+        {
+            "ids": list(event_manager.get_ids()),
+            "timed_out_ids": list(event_manager.timed_out_ids),
+        }
+    )
 
 
-@app.route('/clear_history', methods=['POST'])
+@app.route("/clear_history", methods=["POST"])
 def clear_history():
     temp_em = EventManager()
     temp_em.clear_history()
     temp_em.close_connection()
-    return jsonify({'message': 'History cleared successfully'}), 200
+    return jsonify({"message": "History cleared successfully"}), 200
 
 
 async def check_for_lines_outside_replay_buffer():
-    time_window = datetime.datetime.now() - datetime.timedelta(seconds=gsm_state.replay_buffer_length) - datetime.timedelta(seconds=5)
+    time_window = (
+        datetime.datetime.now()
+        - datetime.timedelta(seconds=gsm_state.replay_buffer_length)
+        - datetime.timedelta(seconds=5)
+    )
     # logger.info(f"Checking for lines outside replay buffer time window: {time_window}")
-    lines_outside_buffer = [line.id for line in event_manager.get_events() if line.time < time_window]
+    lines_outside_buffer = [
+        line.id for line in event_manager.get_events() if line.time < time_window
+    ]
     # logger.info(f"Lines outside replay buffer: {lines_outside_buffer}")
     event_manager.remove_lines_by_ids(lines_outside_buffer, timed_out=True)
-    
+
 
 async def add_event_to_texthooker(line):
     new_event = event_manager.add_gameline(line)
-    await websocket_server_thread.send_text({
-        'event': 'text_received',
-        'sentence': line.text,
-        'data': new_event.to_serializable()
-    })
+    await websocket_server_thread.send_text(
+        {
+            "event": "text_received",
+            "sentence": line.text,
+            "data": new_event.to_serializable(),
+        }
+    )
     if get_config().advanced.plaintext_websocket_port:
         await plaintext_websocket_server_thread.send_text(line.text)
     await check_for_lines_outside_replay_buffer()
@@ -177,61 +288,71 @@ async def send_word_coordinates_to_overlay(boxes):
         await overlay_server_thread.send_text(boxes)
 
 
-@app.route('/update_checkbox', methods=['POST'])
+@app.route("/update_checkbox", methods=["POST"])
 def update_event():
     data = request.get_json()
-    event_id = data.get('id')
+    event_id = data.get("id")
 
     if event_id is None:
-        return jsonify({'error': 'Missing id'}), 400
+        return jsonify({"error": "Missing id"}), 400
     event = event_manager.get(event_id)
     event_manager.get(event_id).checked = not event.checked
-    return jsonify({'message': 'Event updated successfully'}), 200
+    return jsonify({"message": "Event updated successfully"}), 200
 
 
-@app.route('/get-screenshot', methods=['Post'])
+@app.route("/get-screenshot", methods=["Post"])
 def get_screenshot():
     """Endpoint to get a screenshot of the current game screen."""
     data = request.get_json()
-    event_id = data.get('id')
+    event_id = data.get("id")
     if event_id is None:
-        return jsonify({'error': 'Missing id'}), 400
+        return jsonify({"error": "Missing id"}), 400
     line = get_line_by_id(event_id)
     if not line:
-        return jsonify({'error': 'Invalid id'}), 400
+        return jsonify({"error": "Invalid id"}), 400
     gsm_state.line_for_screenshot = line
-    if gsm_state.previous_line_for_screenshot and gsm_state.line_for_screenshot == gsm_state.previous_line_for_screenshot or gsm_state.previous_line_for_audio and gsm_state.line_for_screenshot == gsm_state.previous_line_for_audio:
+    if (
+        gsm_state.previous_line_for_screenshot
+        and gsm_state.line_for_screenshot == gsm_state.previous_line_for_screenshot
+        or gsm_state.previous_line_for_audio
+        and gsm_state.line_for_screenshot == gsm_state.previous_line_for_audio
+    ):
         handle_texthooker_button(gsm_state.previous_replay)
     else:
         obs.save_replay_buffer()
     return jsonify({}), 200
 
 
-@app.route('/play-audio', methods=['POST'])
+@app.route("/play-audio", methods=["POST"])
 def play_audio():
     """Endpoint to play audio for a specific event."""
     data = request.get_json()
-    event_id = data.get('id')
+    event_id = data.get("id")
     if event_id is None:
-        return jsonify({'error': 'Missing id'}), 400
+        return jsonify({"error": "Missing id"}), 400
     print(f"Playing audio for event ID: {event_id}")
     line = get_line_by_id(event_id)
     if not line:
-        return jsonify({'error': 'Invalid id'}), 400
+        return jsonify({"error": "Invalid id"}), 400
     gsm_state.line_for_audio = line
     print(f"gsm_state.line_for_audio: {gsm_state.line_for_audio}")
-    if gsm_state.previous_line_for_audio and gsm_state.line_for_audio == gsm_state.previous_line_for_audio or gsm_state.previous_line_for_screenshot and gsm_state.line_for_audio == gsm_state.previous_line_for_screenshot:
+    if (
+        gsm_state.previous_line_for_audio
+        and gsm_state.line_for_audio == gsm_state.previous_line_for_audio
+        or gsm_state.previous_line_for_screenshot
+        and gsm_state.line_for_audio == gsm_state.previous_line_for_screenshot
+    ):
         handle_texthooker_button(gsm_state.previous_replay)
     else:
         obs.save_replay_buffer()
     return jsonify({}), 200
 
 
-@app.route("/translate-line", methods=['POST'])
+@app.route("/translate-line", methods=["POST"])
 def translate_line():
     data = request.get_json()
-    event_id = data.get('id')
-    text = data.get('text', '').strip()
+    event_id = data.get("id")
+    text = data.get("text", "").strip()
     if event_id is None:
         return jsonify({'error': 'Missing id'}), 400
     
@@ -254,34 +375,46 @@ def translate_line():
         """
 
     if not get_config().ai.is_configured():
-        return jsonify({'error': 'AI translation is not properly configured. Please check your settings in the "AI" Tab.'}), 400
+        return jsonify(
+            {
+                "error": 'AI translation is not properly configured. Please check your settings in the "AI" Tab.'
+            }
+        ), 400
     line = get_line_by_id(event_id)
     if line is None:
-        return jsonify({'error': 'Invalid id'}), 400
+        return jsonify({"error": "Invalid id"}), 400
     line_to_translate = text if text else line.text
     translation = get_ai_prompt_result(
         get_all_lines(), line_to_translate, line, get_current_game(), custom_prompt=prompt
     )
     line.set_TL(translation)
-    return jsonify({'TL': translation}), 200
+    return jsonify({"TL": translation}), 200
+
 
-@app.route('/translate-multiple', methods=['POST'])
+@app.route("/translate-multiple", methods=["POST"])
 def translate_multiple():
     data = request.get_json()
-    event_ids = data.get('ids', [])
+    event_ids = data.get("ids", [])
     if not event_ids:
-        return jsonify({'error': 'Missing ids'}), 400
-    
-    if not get_config().ai.is_configured():
-        return jsonify({'error': 'AI translation is not properly configured. Please check your settings in the "AI" Tab.'}), 400
+        return jsonify({"error": "Missing ids"}), 400
 
-    lines = [get_line_by_id(event_id) for event_id in event_ids if get_line_by_id(event_id) is not None]
+    if not get_config().ai.is_configured():
+        return jsonify(
+            {
+                "error": 'AI translation is not properly configured. Please check your settings in the "AI" Tab.'
+            }
+        ), 400
+
+    lines = [
+        get_line_by_id(event_id)
+        for event_id in event_ids
+        if get_line_by_id(event_id) is not None
+    ]
 
     text = "\n".join(line.text for line in lines)
     
     language = get_config().general.get_native_language_name() if get_config().general.native_language else "English"
     
-    
     translate_multiple_lines_prompt = f"""
     **Professional Game Localization Task**
     Translate the following lines of game dialogue into natural-sounding, context-aware {language}:
@@ -296,62 +429,87 @@ def translate_multiple():
     **Lines to Translate:**
     """
 
-    translation = get_ai_prompt_result(get_all_lines(), text,
-                                        lines[0], get_current_game(), custom_prompt=translate_multiple_lines_prompt)
+    translation = get_ai_prompt_result(
+        get_all_lines(),
+        text,
+        lines[0],
+        get_current_game(),
+        custom_prompt=translate_multiple_lines_prompt,
+    )
 
     return translation, 200
 
-@app.route('/get_status', methods=['GET'])
+
+@app.route("/get_status", methods=["GET"])
 def get_status():
     return jsonify(gsm_status.to_dict()), 200
 
-@app.template_filter('datetimeformat')
-def datetimeformat(value, format='%Y-%m-%d %H:%M:%S'):
+
+@app.template_filter("datetimeformat")
+def datetimeformat(value, format="%Y-%m-%d %H:%M:%S"):
     """Formats a timestamp into a human-readable string."""
     if value is None:
         return ""
     return datetime.datetime.fromtimestamp(float(value)).strftime(format)
 
 
-@app.route('/overview')
+@app.route("/overview")
 def overview():
     """Renders the overview page."""
     from GameSentenceMiner.util.configuration import get_master_config, get_stats_config
-    return render_template('overview.html',
-                         config=get_config(),
-                         master_config=get_master_config(),
-                         stats_config=get_stats_config())
 
-@app.route('/stats')
+    return render_template(
+        "overview.html",
+        config=get_config(),
+        master_config=get_master_config(),
+        stats_config=get_stats_config(),
+    )
+
+
+@app.route("/stats")
 def stats():
     """Renders the stats page."""
     from GameSentenceMiner.util.configuration import get_master_config, get_stats_config
-    return render_template('stats.html',
-                         config=get_config(),
-                         master_config=get_master_config(),
-                         stats_config=get_stats_config())
+    from GameSentenceMiner.util.stats_rollup_table import StatsRollupTable
+
+    # Get first date from rollup table to avoid extra API call on page load
+    first_rollup_date = StatsRollupTable.get_first_date()
+
+    return render_template(
+        "stats.html",
+        config=get_config(),
+        master_config=get_master_config(),
+        stats_config=get_stats_config(),
+        first_rollup_date=first_rollup_date,
+    )
 
-@app.route('/goals')
+
+@app.route("/goals")
 def goals():
     """Renders the goals page."""
     from GameSentenceMiner.util.configuration import get_master_config, get_stats_config
-    return render_template('goals.html',
-                         config=get_config(),
-                         master_config=get_master_config(),
-                         stats_config=get_stats_config())
+
+    return render_template(
+        "goals.html",
+        config=get_config(),
+        master_config=get_master_config(),
+        stats_config=get_stats_config(),
+    )
 
 
-@app.route('/search')
+@app.route("/search")
 def search():
     """Renders the search page."""
-    return render_template('search.html')
+    return render_template("search.html")
 
-@app.route('/anki_stats')
+
+@app.route("/anki_stats")
 def anki_stats():
     """Renders the Anki statistics page."""
-    return render_template('anki_stats.html')
+    return render_template("anki_stats.html")
+
 
-@app.route('/get_websocket_port', methods=['GET'])
+@app.route("/get_websocket_port", methods=["GET"])
 def get_websocket_port():
     return jsonify({"port": websocket_server_thread.get_ws_port_func()}), 200
 
@@ -366,28 +524,36 @@ def are_lines_selected():
 
 def reset_checked_lines():
     async def send_reset_message():
-        await websocket_server_thread.send_text({
-            'event': 'reset_checkboxes',
-        })
+        await websocket_server_thread.send_text(
+            {
+                "event": "reset_checkboxes",
+            }
+        )
+
     event_manager.reset_checked_lines()
     asyncio.run(send_reset_message())
-    
+
+
 def reset_buttons():
     async def send_reset_message():
-        await websocket_server_thread.send_text({
-            'event': 'reset_buttons',
-        })
+        await websocket_server_thread.send_text(
+            {
+                "event": "reset_buttons",
+            }
+        )
+
     asyncio.run(send_reset_message())
 
 
 def open_texthooker():
-    webbrowser.open(url + '/texthooker')
+    webbrowser.open(url + "/texthooker")
 
 
 def start_web_server():
     logger.debug("Starting web server...")
     import logging
-    log = logging.getLogger('werkzeug')
+
+    log = logging.getLogger("werkzeug")
     log.setLevel(logging.ERROR)  # Set to ERROR to suppress most logs
 
     # Open the default browser
@@ -401,7 +567,10 @@ def start_web_server():
 
 
 async def texthooker_page_coro():
-    global websocket_server_thread, plaintext_websocket_server_thread, overlay_server_thread
+    global \
+        websocket_server_thread, \
+        plaintext_websocket_server_thread, \
+        overlay_server_thread
     # Run the WebSocket server in the asyncio event loop
     flask_thread = threading.Thread(target=start_web_server)
     flask_thread.daemon = True
@@ -417,5 +586,5 @@ def run_text_hooker_page():
         logger.info("Shutting down due to KeyboardInterrupt.")
 
 
-if __name__ == '__main__':
-    asyncio.run(texthooker_page_coro())
+if __name__ == "__main__":
+    asyncio.run(texthooker_page_coro())