GameSentenceMiner 2.17.7__py3-none-any.whl → 2.18.1__py3-none-any.whl

GameSentenceMiner/web/database_api.py

@@ -3,8 +3,11 @@ import datetime
 import re
 import csv
 import io
+import os
+import json
 from collections import defaultdict
 import time
+from pathlib import Path
 
 import flask
 from flask import request, jsonify
@@ -212,7 +215,7 @@ def register_database_api_routes(app):
             return jsonify({'games': games_data}), 200
             
         except Exception as e:
-            logger.error(f"Error fetching games list: {e}")
+            logger.error(f"Error fetching games list: {e}", exc_info=True)
             return jsonify({'error': 'Failed to fetch games list'}), 500
 
     @app.route('/api/delete-games', methods=['POST'])
@@ -306,6 +309,9 @@ def register_database_api_routes(app):
                 'reading_hours_target': config.reading_hours_target,
                 'character_count_target': config.character_count_target,
                 'games_target': config.games_target,
+                'reading_hours_target_date': config.reading_hours_target_date,
+                'character_count_target_date': config.character_count_target_date,
+                'games_target_date': config.games_target_date,
             }), 200
         except Exception as e:
             logger.error(f"Error getting settings: {e}")
@@ -328,6 +334,9 @@ def register_database_api_routes(app):
             reading_hours_target = data.get('reading_hours_target')
             character_count_target = data.get('character_count_target')
             games_target = data.get('games_target')
+            reading_hours_target_date = data.get('reading_hours_target_date')
+            character_count_target_date = data.get('character_count_target_date')
+            games_target_date = data.get('games_target_date')
             
             # Validate input - only require the settings that are provided
             settings_to_update = {}
@@ -386,6 +395,37 @@ def register_database_api_routes(app):
                 except (ValueError, TypeError):
                     return jsonify({'error': 'Games target must be a valid integer'}), 400
             
+            # Validate target dates (ISO format: YYYY-MM-DD)
+            if reading_hours_target_date is not None:
+                if reading_hours_target_date == '':
+                    settings_to_update['reading_hours_target_date'] = ''
+                else:
+                    try:
+                        datetime.datetime.strptime(reading_hours_target_date, '%Y-%m-%d')
+                        settings_to_update['reading_hours_target_date'] = reading_hours_target_date
+                    except ValueError:
+                        return jsonify({'error': 'Reading hours target date must be in YYYY-MM-DD format'}), 400
+            
+            if character_count_target_date is not None:
+                if character_count_target_date == '':
+                    settings_to_update['character_count_target_date'] = ''
+                else:
+                    try:
+                        datetime.datetime.strptime(character_count_target_date, '%Y-%m-%d')
+                        settings_to_update['character_count_target_date'] = character_count_target_date
+                    except ValueError:
+                        return jsonify({'error': 'Character count target date must be in YYYY-MM-DD format'}), 400
+            
+            if games_target_date is not None:
+                if games_target_date == '':
+                    settings_to_update['games_target_date'] = ''
+                else:
+                    try:
+                        datetime.datetime.strptime(games_target_date, '%Y-%m-%d')
+                        settings_to_update['games_target_date'] = games_target_date
+                    except ValueError:
+                        return jsonify({'error': 'Games target date must be in YYYY-MM-DD format'}), 400
+            
             if not settings_to_update:
                 return jsonify({'error': 'No valid settings provided'}), 400
             
@@ -404,6 +444,12 @@ def register_database_api_routes(app):
                 config.character_count_target = settings_to_update['character_count_target']
             if 'games_target' in settings_to_update:
                 config.games_target = settings_to_update['games_target']
+            if 'reading_hours_target_date' in settings_to_update:
+                config.reading_hours_target_date = settings_to_update['reading_hours_target_date']
+            if 'character_count_target_date' in settings_to_update:
+                config.character_count_target_date = settings_to_update['character_count_target_date']
+            if 'games_target_date' in settings_to_update:
+                config.games_target_date = settings_to_update['games_target_date']
             
             save_stats_config(config)
 
@@ -786,146 +832,596 @@ def register_database_api_routes(app):
             logger.error(f"Error in deduplication: {e}")
             return jsonify({'error': f'Deduplication failed: {str(e)}'}), 500
 
+    @app.route('/api/merge_games', methods=['POST'])
+    def api_merge_games():
+        """
+        Merges multiple selected games into a single game entry.
+        The first game in the list becomes the primary game that retains its name.
+        All lines from secondary games are moved to the primary game.
+        """
+        try:
+            data = request.get_json()
+            target_game = data.get('target_game', None)
+            games_to_merge = data.get('games_to_merge', [])
+            
+            logger.info(f"Merge request received: target_game='{target_game}', games_to_merge={games_to_merge}")
+            
+            # Validation
+            if not target_game:
+                return jsonify({'error': 'No target game specified for merging'}), 400
+
+            if not games_to_merge:
+                return jsonify({'error': 'No games specified for merging'}), 400
+            
+            if not isinstance(games_to_merge, list):
+                return jsonify({'error': 'game_names must be a list'}), 400
+                
+            if len(games_to_merge) < 1:
+                return jsonify({'error': 'At least 1 game must be selected for merging'}), 400
+
+            # Validate that all games exist
+            existing_games = GameLinesTable.get_all_games_with_lines()
+            invalid_games = [name for name in games_to_merge if name not in existing_games]
+            
+            if invalid_games:
+                return jsonify({'error': f'Games not found: {", ".join(invalid_games)}'}), 400
+            
+            # Check for duplicate game names
+            if len(set(games_to_merge)) != len(games_to_merge):
+                return jsonify({'error': 'Duplicate game names found in selection'}), 400
+            
+            # Identify primary and secondary games
+
+            # Collect pre-merge statistics
+            primary_lines_before = GameLinesTable.get_all_lines_for_scene(target_game)
+            total_lines_to_merge = 0
+            merge_summary = {
+                'primary_game': target_game,
+                'secondary_games': games_to_merge,
+                'lines_moved': 0,
+                'total_lines_after_merge': 0
+            }
+            
+            # Calculate lines to be moved and store counts
+            secondary_game_line_counts = {}
+            for game_name in games_to_merge:
+                secondary_lines = GameLinesTable.get_all_lines_for_scene(game_name)
+                line_count = len(secondary_lines)
+                secondary_game_line_counts[game_name] = line_count
+                total_lines_to_merge += line_count
+            
+            if total_lines_to_merge == 0:
+                return jsonify({'error': 'No lines found in secondary games to merge'}), 400
+            
+            # Begin database transaction for merge
+            try:
+                # Perform the merge operation within transaction
+                lines_moved = 0
+                for game_name in games_to_merge:
+                    # Update game_name for all lines belonging to this secondary game
+                    # Also set original_game_name to preserve the original title
+                    # Ensure the table name is as expected to prevent SQL injection
+                    if GameLinesTable._table != "game_lines":
+                        raise ValueError("Unexpected table name in GameLinesTable._table")
+                    GameLinesTable._db.execute(
+                        "UPDATE game_lines SET game_name=?, original_game_name=COALESCE(original_game_name, ?) WHERE game_name=?",
+                        (target_game, game_name, game_name),
+                        commit=True
+                    )
+                    
+                    # Add the count we calculated earlier
+                    lines_moved += secondary_game_line_counts[game_name]
+                
+                # Update merge summary
+                merge_summary['lines_moved'] = lines_moved
+                merge_summary['total_lines_after_merge'] = len(primary_lines_before) + lines_moved
+                
+                # Log the successful merge
+                logger.info(f"Successfully merged {len(games_to_merge)} games into '{target_game}': moved {lines_moved} lines")
+
+                # Prepare success response
+                response_data = {
+                    'message': f'Successfully merged {len(games_to_merge)} games into "{target_game}"',
+                    'primary_game': target_game,
+                    'merged_games': games_to_merge,
+                    'lines_moved': lines_moved,
+                    'total_lines_in_primary': merge_summary['total_lines_after_merge'],
+                    'merge_summary': merge_summary
+                }
+                
+                return jsonify(response_data), 200
+                
+            except Exception as db_error:
+                logger.error(f"Database error during game merge: {db_error}", exc_info=True)
+                return jsonify({
+                    'error': f'Failed to merge games due to database error: {str(db_error)}'
+                }), 500
+                
+        except Exception as e:
+            logger.error(f"Error in game merge API: {e}")
+            return jsonify({'error': f'Game merge failed: {str(e)}'}), 500
+
     @app.route('/api/stats')
     def api_stats():
         """
         Provides aggregated, cumulative stats for charting.
         Accepts optional 'year' parameter to filter heatmap data.
         """
-        punctionation_regex = regex.compile(r'[\p{P}\p{S}\p{Z}]')
-        # Get optional year filter parameter
-        filter_year = request.args.get('year', None)
+        try:
+            punctionation_regex = regex.compile(r'[\p{P}\p{S}\p{Z}]')
+            # Get optional year filter parameter
+            filter_year = request.args.get('year', None)
 
-        # Get Start and End time as unix timestamp
-        start_timestamp = request.args.get('start', None)
-        end_timestamp = request.args.get('end', None)
-        
-        # Convert timestamps to float if provided
-        start_timestamp = float(start_timestamp) if start_timestamp else None
-        end_timestamp = float(end_timestamp) if end_timestamp else None
+            # Get Start and End time as unix timestamp
+            start_timestamp = request.args.get('start', None)
+            end_timestamp = request.args.get('end', None)
+            
+            # Convert timestamps to float if provided
+            start_timestamp = float(start_timestamp) if start_timestamp else None
+            end_timestamp = float(end_timestamp) if end_timestamp else None
 
-        # # 1. Fetch all lines and sort them chronologically
-        # all_lines = sorted(GameLinesTable.all(), key=lambda line: line.timestamp)
+            # 1. Fetch all lines and sort them chronologically
+            all_lines = GameLinesTable.get_lines_filtered_by_timestamp(start=start_timestamp, end=end_timestamp)
+            
+            if not all_lines:
+                return jsonify({"labels": [], "datasets": []})
+
+            # 2. Process data into daily totals for each game
+            # Structure: daily_data[date_str][game_name] = {'lines': N, 'chars': N}
+            daily_data = defaultdict(lambda: defaultdict(lambda: {'lines': 0, 'chars': 0}))
+            wrong_instance_found = False
+            for line in all_lines:
+                day_str = datetime.date.fromtimestamp(float(line.timestamp)).strftime('%Y-%m-%d')
+                game = line.game_name or "Unknown Game"
+                # Remove punctuation and symbols from line text before counting characters
+                clean_text = punctionation_regex.sub('', str(line.line_text)) if line.line_text else ''
+                if not isinstance(clean_text, str) and not wrong_instance_found:
+                    logger.info(f"Non-string line_text encountered: {clean_text} (type: {type(clean_text)})")
+                    wrong_instance_found = True
+
+                line.line_text = clean_text  # Update line text to cleaned version for future use
+                daily_data[day_str][game]['lines'] += 1
+                daily_data[day_str][game]['chars'] += len(clean_text)
+
+            # 3. Create cumulative datasets for Chart.js
+            sorted_days = sorted(daily_data.keys())
+            game_names = GameLinesTable.get_all_games_with_lines()
+            
+            # Keep track of the running total for each metric for each game
+            cumulative_totals = defaultdict(lambda: {'lines': 0, 'chars': 0})
+            
+            # Structure for final data: final_data[game_name][metric] = [day1_val, day2_val, ...]
+            final_data = defaultdict(lambda: defaultdict(list))
 
-        # 1. Fetch all lines and sort them chronologically
-        all_lines = GameLinesTable.get_lines_filtered_by_timestamp(start=start_timestamp, end=end_timestamp)
-        
-        if not all_lines:
-            return jsonify({"labels": [], "datasets": []})
+            for day in sorted_days:
+                for game in game_names:
+                    # Add the day's total to the cumulative total
+                    cumulative_totals[game]['lines'] += daily_data[day][game]['lines']
+                    cumulative_totals[game]['chars'] += daily_data[day][game]['chars']
+                    
+                    # Append the new cumulative total to the list for that day
+                    final_data[game]['lines'].append(cumulative_totals[game]['lines'])
+                    final_data[game]['chars'].append(cumulative_totals[game]['chars'])
+            
+            # 4. Format into Chart.js dataset structure
+            datasets = []
+            # A simple color palette for the chart lines
+            colors = ['#3498db', '#e74c3c', '#2ecc71', '#f1c40f', '#9b59b6', '#1abc9c', '#e67e22']
+            
+            for i, game in enumerate(game_names):
+                color = colors[i % len(colors)]
+            
+            # 1. Fetch all lines and sort them chronologically
+            try:
+                all_lines = sorted(GameLinesTable.all(), key=lambda line: line.timestamp)
+            except Exception as e:
+                logger.error(f"Error fetching lines from database: {e}")
+                return jsonify({'error': 'Failed to fetch data from database'}), 500
+            
+            if not all_lines:
+                return jsonify({"labels": [], "datasets": []})
 
-        # 2. Process data into daily totals for each game
-        # Structure: daily_data[date_str][game_name] = {'lines': N, 'chars': N}
-        daily_data = defaultdict(lambda: defaultdict(lambda: {'lines': 0, 'chars': 0}))
-        
-        
-        
-        
-        # start_time = time.perf_counter()
-        # for line in all_lines:
-        #     day_str = datetime.date.fromtimestamp(float(line.timestamp)).strftime('%Y-%m-%d')
-        #     game = line.game_name or "Unknown Game"
-        #     daily_data[day_str][game]['lines'] += 1
-        #     daily_data[day_str][game]['chars'] += len(line.line_text) if line.line_text else 0
-        # end_time = time.perf_counter()
-        # logger.info(f"Without Punctuation removal and daily aggregation took {end_time - start_time:.4f} seconds for {len(all_lines)} lines")
+            # 2. Process data into daily totals for each game
+            # Structure: daily_data[date_str][game_name] = {'lines': N, 'chars': N}
+            daily_data = defaultdict(lambda: defaultdict(lambda: {'lines': 0, 'chars': 0}))
 
-        # start_time = time.perf_counter()
-        wrong_instance_found = False
-        for line in all_lines:
-            day_str = datetime.date.fromtimestamp(float(line.timestamp)).strftime('%Y-%m-%d')
-            game = line.game_name or "Unknown Game"
-            # Remove punctuation and symbols from line text before counting characters
-            clean_text = punctionation_regex.sub('', str(line.line_text)) if line.line_text else ''
-            if not isinstance(clean_text, str) and not wrong_instance_found:
-                logger.info(f"Non-string line_text encountered: {clean_text} (type: {type(clean_text)})")
-                wrong_instance_found = True
+            try:
+                # start_time = time.perf_counter()
+                # for line in all_lines:
+                #     day_str = datetime.date.fromtimestamp(float(line.timestamp)).strftime('%Y-%m-%d')
+                #     game = line.game_name or "Unknown Game"
+                #     daily_data[day_str][game]['lines'] += 1
+                #     daily_data[day_str][game]['chars'] += len(line.line_text) if line.line_text else 0
+                # end_time = time.perf_counter()
+                # logger.info(f"Without Punctuation removal and daily aggregation took {end_time - start_time:.4f} seconds for {len(all_lines)} lines")
 
-            line.line_text = clean_text  # Update line text to cleaned version for future use
-            daily_data[day_str][game]['lines'] += 1
-            daily_data[day_str][game]['chars'] += len(clean_text)
-        # end_time = time.perf_counter()
-        # logger.info(f"With Punctuation removal and daily aggregation took {end_time - start_time:.4f} seconds for {len(all_lines)} lines")
+                # start_time = time.perf_counter()
+                wrong_instance_found = False
+                for line in all_lines:
+                    day_str = datetime.date.fromtimestamp(float(line.timestamp)).strftime('%Y-%m-%d')
+                    game = line.game_name or "Unknown Game"
+                    # Remove punctuation and symbols from line text before counting characters
+                    clean_text = punctionation_regex.sub('', str(line.line_text)) if line.line_text else ''
+                    if not isinstance(clean_text, str) and not wrong_instance_found:
+                        logger.info(f"Non-string line_text encountered: {clean_text} (type: {type(clean_text)})")
+                        wrong_instance_found = True
 
-        # 3. Create cumulative datasets for Chart.js
-        sorted_days = sorted(daily_data.keys())
-        game_names = GameLinesTable.get_all_games_with_lines()
-        
-        # Keep track of the running total for each metric for each game
-        cumulative_totals = defaultdict(lambda: {'lines': 0, 'chars': 0})
-        
-        # Structure for final data: final_data[game_name][metric] = [day1_val, day2_val, ...]
-        final_data = defaultdict(lambda: defaultdict(list))
+                    line.line_text = clean_text  # Update line text to cleaned version for future use
+                    daily_data[day_str][game]['lines'] += 1
+                    daily_data[day_str][game]['chars'] += len(clean_text)
+                # end_time = time.perf_counter()
+                # logger.info(f"With Punctuation removal and daily aggregation took {end_time - start_time:.4f} seconds for {len(all_lines)} lines")
+            except Exception as e:
+                logger.error(f"Error processing daily data: {e}")
+                return jsonify({'error': 'Failed to process daily data'}), 500
 
-        for day in sorted_days:
-            for game in game_names:
-                # Add the day's total to the cumulative total
-                cumulative_totals[game]['lines'] += daily_data[day][game]['lines']
-                cumulative_totals[game]['chars'] += daily_data[day][game]['chars']
+            # 3. Create cumulative datasets for Chart.js
+            try:
+                sorted_days = sorted(daily_data.keys())
+                game_names = GameLinesTable.get_all_games_with_lines()
                 
-                # Append the new cumulative total to the list for that day
-                final_data[game]['lines'].append(cumulative_totals[game]['lines'])
-                final_data[game]['chars'].append(cumulative_totals[game]['chars'])
-        
-        # 4. Format into Chart.js dataset structure
-        datasets = []
-        # A simple color palette for the chart lines
-        colors = ['#3498db', '#e74c3c', '#2ecc71', '#f1c40f', '#9b59b6', '#1abc9c', '#e67e22']
-        
-        for i, game in enumerate(game_names):
-            color = colors[i % len(colors)]
-            
-            datasets.append({
-                "label": f"{game}",
-                "for": "Lines Received",
-                "data": final_data[game]['lines'],
-                "borderColor": color,
-                "backgroundColor": f"{color}33", # Semi-transparent for fill
-                "fill": False,
-                "tension": 0.1
-            })
-            datasets.append({
-                "label": f"{game}",
-                "for": "Characters Read",
-                "data": final_data[game]['chars'],
-                "borderColor": color,
-                "backgroundColor": f"{color}33",
-                "fill": False,
-                "tension": 0.1,
-                "hidden": True # Hide by default to not clutter the chart
-            })
+                # Keep track of the running total for each metric for each game
+                cumulative_totals = defaultdict(lambda: {'lines': 0, 'chars': 0})
+                
+                # Structure for final data: final_data[game_name][metric] = [day1_val, day2_val, ...]
+                final_data = defaultdict(lambda: defaultdict(list))
+
+                for day in sorted_days:
+                    for game in game_names:
+                        # Add the day's total to the cumulative total
+                        cumulative_totals[game]['lines'] += daily_data[day][game]['lines']
+                        cumulative_totals[game]['chars'] += daily_data[day][game]['chars']
+                        
+                        # Append the new cumulative total to the list for that day
+                        final_data[game]['lines'].append(cumulative_totals[game]['lines'])
+                        final_data[game]['chars'].append(cumulative_totals[game]['chars'])
+            except Exception as e:
+                logger.error(f"Error creating cumulative datasets: {e}")
+                return jsonify({'error': 'Failed to create datasets'}), 500
+            
+            # 4. Format into Chart.js dataset structure
+            try:
+                datasets = []
+                # A simple color palette for the chart lines
+                colors = ['#3498db', '#e74c3c', '#2ecc71', '#f1c40f', '#9b59b6', '#1abc9c', '#e67e22']
+                
+                for i, game in enumerate(game_names):
+                    color = colors[i % len(colors)]
+                    
+                    datasets.append({
+                        "label": f"{game} - Lines Received",
+                        "data": final_data[game]['lines'],
+                        "borderColor": color,
+                        "backgroundColor": f"{color}33", # Semi-transparent for fill
+                        "fill": False,
+                        "tension": 0.1,
+                        "for": "Lines Received"
+                    })
+                    datasets.append({
+                        "label": f"{game} - Characters Read",
+                        "data": final_data[game]['chars'],
+                        "borderColor": color,
+                        "backgroundColor": f"{color}33",
+                        "fill": False,
+                        "tension": 0.1,
+                        "hidden": True, # Hide by default to not clutter the chart
+                        "for": "Characters Read"
+                    })
+            except Exception as e:
+                logger.error(f"Error formatting Chart.js datasets: {e}")
+                return jsonify({'error': 'Failed to format chart data'}), 500
+
+            # 5. Calculate additional chart data
+            try:
+                kanji_grid_data = calculate_kanji_frequency(all_lines)
+            except Exception as e:
+                logger.error(f"Error calculating kanji frequency: {e}")
+                kanji_grid_data = []
+                
+            try:
+                heatmap_data = calculate_heatmap_data(all_lines, filter_year)
+            except Exception as e:
+                logger.error(f"Error calculating heatmap data: {e}")
+                heatmap_data = []
+                
+            try:
+                total_chars_data = calculate_total_chars_per_game(all_lines)
+            except Exception as e:
+                logger.error(f"Error calculating total chars per game: {e}")
+                total_chars_data = {}
+                
+            try:
+                reading_time_data = calculate_reading_time_per_game(all_lines)
+            except Exception as e:
+                logger.error(f"Error calculating reading time per game: {e}")
+                reading_time_data = {}
+                
+            try:
+                reading_speed_per_game_data = calculate_reading_speed_per_game(all_lines)
+            except Exception as e:
+                logger.error(f"Error calculating reading speed per game: {e}")
+                reading_speed_per_game_data = {}
+            
+            # 6. Calculate dashboard statistics
+            try:
+                current_game_stats = calculate_current_game_stats(all_lines)
+            except Exception as e:
+                logger.error(f"Error calculating current game stats: {e}")
+                current_game_stats = {}
+                
+            try:
+                all_games_stats = calculate_all_games_stats(all_lines)
+            except Exception as e:
+                logger.error(f"Error calculating all games stats: {e}")
+                all_games_stats = {}
 
-        # 5. Calculate additional chart data
-        kanji_grid_data = calculate_kanji_frequency(all_lines)
-        heatmap_data = calculate_heatmap_data(all_lines, filter_year)
-        total_chars_data = calculate_total_chars_per_game(all_lines)
-        reading_time_data = calculate_reading_time_per_game(all_lines)
-        reading_speed_per_game_data = calculate_reading_speed_per_game(all_lines)
-        
-        # 6. Calculate dashboard statistics
-        current_game_stats = calculate_current_game_stats(all_lines)
-        all_games_stats = calculate_all_games_stats(all_lines)
+            # 7. Prepare allLinesData for frontend calculations (needed for average daily time)
+            try:
+                all_lines_data = []
+                for line in all_lines:
+                    all_lines_data.append({
+                        'timestamp': float(line.timestamp),
+                        'game_name': line.game_name or 'Unknown Game',
+                        'characters': len(line.line_text) if line.line_text else 0
+                    })
+            except Exception as e:
+                logger.error(f"Error preparing all lines data: {e}")
+                all_lines_data = []
 
-        # 7. Prepare allLinesData for frontend calculations (needed for average daily time)
-        all_lines_data = []
-        for line in all_lines:
-            all_lines_data.append({
-                'timestamp': float(line.timestamp),
-                'game_name': line.game_name or 'Unknown Game',
-                'characters': len(line.line_text) if line.line_text else 0
+            return jsonify({
+                "labels": sorted_days,
+                "datasets": datasets,
+                "kanjiGridData": kanji_grid_data,
+                "heatmapData": heatmap_data,
+                "totalCharsPerGame": total_chars_data,
+                "readingTimePerGame": reading_time_data,
+                "readingSpeedPerGame": reading_speed_per_game_data,
+                "currentGameStats": current_game_stats,
+                "allGamesStats": all_games_stats,
+                "allLinesData": all_lines_data
             })
+            
+        except Exception as e:
+            logger.error(f"Unexpected error in api_stats: {e}")
+            return jsonify({'error': 'Failed to generate statistics'}), 500
+
+    @app.route('/api/goals-today', methods=['GET'])
+    def api_goals_today():
+        """
+        Calculate daily requirements and current progress for today based on goal target dates.
+        Returns what needs to be accomplished today to stay on track.
+        """
+        try:
+            config = get_stats_config()
+            today = datetime.date.today()
+            
+            # Get all lines for overall progress
+            all_lines = GameLinesTable.all()
+            if not all_lines:
+                return jsonify({
+                    'hours': {'required': 0, 'progress': 0, 'has_target': False},
+                    'characters': {'required': 0, 'progress': 0, 'has_target': False},
+                    'games': {'required': 0, 'progress': 0, 'has_target': False}
+                }), 200
+            
+            # Calculate overall current progress
+            timestamps = [float(line.timestamp) for line in all_lines]
+            total_time_seconds = calculate_actual_reading_time(timestamps)
+            total_hours = total_time_seconds / 3600
+            total_characters = sum(len(line.line_text) if line.line_text else 0 for line in all_lines)
+            total_games = len(set(line.game_name or "Unknown Game" for line in all_lines))
+            
+            # Get today's lines for progress
+            today_lines = [line for line in all_lines 
+                          if datetime.date.fromtimestamp(float(line.timestamp)) == today]
+            
+            today_timestamps = [float(line.timestamp) for line in today_lines]
+            today_time_seconds = calculate_actual_reading_time(today_timestamps) if len(today_timestamps) >= 2 else 0
+            today_hours = today_time_seconds / 3600
+            today_characters = sum(len(line.line_text) if line.line_text else 0 for line in today_lines)
+            
+            result = {}
+            
+            # Calculate hours requirement
+            if config.reading_hours_target_date:
+                try:
+                    target_date = datetime.datetime.strptime(config.reading_hours_target_date, '%Y-%m-%d').date()
+                    days_remaining = (target_date - today).days + 1  # +1 to include today
+                    if days_remaining > 0:
+                        hours_needed = max(0, config.reading_hours_target - total_hours)
+                        daily_hours_required = hours_needed / days_remaining
+                        result['hours'] = {
+                            'required': round(daily_hours_required, 2),
+                            'progress': round(today_hours, 2),
+                            'has_target': True,
+                            'target_date': config.reading_hours_target_date,
+                            'days_remaining': days_remaining
+                        }
+                    else:
+                        result['hours'] = {'required': 0, 'progress': round(today_hours, 2), 'has_target': True, 'expired': True}
+                except ValueError:
+                    result['hours'] = {'required': 0, 'progress': round(today_hours, 2), 'has_target': False}
+            else:
+                result['hours'] = {'required': 0, 'progress': round(today_hours, 2), 'has_target': False}
+            
+            # Calculate characters requirement
+            if config.character_count_target_date:
+                try:
+                    target_date = datetime.datetime.strptime(config.character_count_target_date, '%Y-%m-%d').date()
+                    days_remaining = (target_date - today).days + 1
+                    if days_remaining > 0:
+                        chars_needed = max(0, config.character_count_target - total_characters)
+                        daily_chars_required = int(chars_needed / days_remaining)
+                        result['characters'] = {
+                            'required': daily_chars_required,
+                            'progress': today_characters,
+                            'has_target': True,
+                            'target_date': config.character_count_target_date,
+                            'days_remaining': days_remaining
+                        }
+                    else:
+                        result['characters'] = {'required': 0, 'progress': today_characters, 'has_target': True, 'expired': True}
+                except ValueError:
+                    result['characters'] = {'required': 0, 'progress': today_characters, 'has_target': False}
+            else:
+                result['characters'] = {'required': 0, 'progress': today_characters, 'has_target': False}
+            
+            # Calculate games requirement
+            if config.games_target_date:
+                try:
+                    target_date = datetime.datetime.strptime(config.games_target_date, '%Y-%m-%d').date()
+                    days_remaining = (target_date - today).days + 1
+                    if days_remaining > 0:
+                        games_needed = max(0, config.games_target - total_games)
+                        daily_games_required = games_needed / days_remaining
+                        result['games'] = {
+                            'required': round(daily_games_required, 2),
+                            'progress': total_games,
+                            'has_target': True,
+                            'target_date': config.games_target_date,
+                            'days_remaining': days_remaining
+                        }
+                    else:
+                        result['games'] = {'required': 0, 'progress': total_games, 'has_target': True, 'expired': True}
+                except ValueError:
+                    result['games'] = {'required': 0, 'progress': total_games, 'has_target': False}
+            else:
+                result['games'] = {'required': 0, 'progress': total_games, 'has_target': False}
+            
+            return jsonify(result), 200
+            
+        except Exception as e:
+            logger.error(f"Error calculating goals today: {e}")
+            return jsonify({'error': 'Failed to calculate daily goals'}), 500
 
-        return jsonify({
-            "labels": sorted_days,
-            "datasets": datasets,
-            "kanjiGridData": kanji_grid_data,
-            "heatmapData": heatmap_data,
-            "totalCharsPerGame": total_chars_data,
-            "readingTimePerGame": reading_time_data,
-            "readingSpeedPerGame": reading_speed_per_game_data,
-            "currentGameStats": current_game_stats,
-            "allGamesStats": all_games_stats,
-            "allLinesData": all_lines_data
-        })
+    @app.route('/api/goals-projection', methods=['GET'])
+    def api_goals_projection():
+        """
+        Calculate projections based on 30-day rolling average.
+        Returns projected stats by target dates.
+        """
+        try:
+            config = get_stats_config()
+            today = datetime.date.today()
+            thirty_days_ago = today - datetime.timedelta(days=30)
+            
+            # Get all lines
+            all_lines = GameLinesTable.all()
+            if not all_lines:
+                return jsonify({
+                    'hours': {'projection': 0, 'daily_average': 0},
+                    'characters': {'projection': 0, 'daily_average': 0},
+                    'games': {'projection': 0, 'daily_average': 0}
+                }), 200
+            
+            # Get last 30 days of lines
+            recent_lines = [line for line in all_lines 
+                           if datetime.date.fromtimestamp(float(line.timestamp)) >= thirty_days_ago]
+            
+            # Calculate 30-day averages
+            if recent_lines:
+                # Group by day for accurate averaging
+                daily_data = defaultdict(lambda: {'timestamps': [], 'characters': 0, 'games': set()})
+                for line in recent_lines:
+                    day_str = datetime.date.fromtimestamp(float(line.timestamp)).strftime('%Y-%m-%d')
+                    daily_data[day_str]['timestamps'].append(float(line.timestamp))
+                    daily_data[day_str]['characters'] += len(line.line_text) if line.line_text else 0
+                    daily_data[day_str]['games'].add(line.game_name or "Unknown Game")
+                
+                # Calculate daily averages
+                total_hours = 0
+                total_chars = 0
+                total_unique_games = set()
+                
+                for day_data in daily_data.values():
+                    if len(day_data['timestamps']) >= 2:
+                        day_seconds = calculate_actual_reading_time(day_data['timestamps'])
+                        total_hours += day_seconds / 3600
+                    total_chars += day_data['characters']
+                    total_unique_games.update(day_data['games'])
+                
+                # Average over ALL 30 days (including days with 0 activity)
+                avg_daily_hours = total_hours / 30
+                avg_daily_chars = total_chars / 30
+                # Calculate average daily unique games correctly
+                today = datetime.date.today()
+                daily_unique_games_counts = []
+                for i in range(30):
+                    day = (today - datetime.timedelta(days=i)).strftime('%Y-%m-%d')
+                    daily_unique_games_counts.append(len(daily_data[day]['games']) if day in daily_data else 0)
+                avg_daily_games = sum(daily_unique_games_counts) / 30
+            else:
+                avg_daily_hours = 0
+                avg_daily_chars = 0
+                avg_daily_games = 0
+            
+            # Calculate current totals
+            timestamps = [float(line.timestamp) for line in all_lines]
+            current_hours = calculate_actual_reading_time(timestamps) / 3600
+            current_chars = sum(len(line.line_text) if line.line_text else 0 for line in all_lines)
+            current_games = len(set(line.game_name or "Unknown Game" for line in all_lines))
+            
+            result = {}
+            
+            # Project hours by target date
+            if config.reading_hours_target_date:
+                try:
+                    target_date = datetime.datetime.strptime(config.reading_hours_target_date, '%Y-%m-%d').date()
+                    days_until_target = (target_date - today).days
+                    projected_hours = current_hours + (avg_daily_hours * days_until_target)
+                    result['hours'] = {
+                        'projection': round(projected_hours, 2),
+                        'daily_average': round(avg_daily_hours, 2),
+                        'target_date': config.reading_hours_target_date,
+                        'target': config.reading_hours_target,
+                        'current': round(current_hours, 2)
+                    }
+                except ValueError:
+                    result['hours'] = {'projection': 0, 'daily_average': round(avg_daily_hours, 2)}
+            else:
+                result['hours'] = {'projection': 0, 'daily_average': round(avg_daily_hours, 2)}
+            
+            # Project characters by target date
+            if config.character_count_target_date:
+                try:
+                    target_date = datetime.datetime.strptime(config.character_count_target_date, '%Y-%m-%d').date()
+                    days_until_target = (target_date - today).days
+                    projected_chars = int(current_chars + (avg_daily_chars * days_until_target))
+                    result['characters'] = {
+                        'projection': projected_chars,
+                        'daily_average': int(avg_daily_chars),
+                        'target_date': config.character_count_target_date,
+                        'target': config.character_count_target,
+                        'current': current_chars
+                    }
+                except ValueError:
+                    result['characters'] = {'projection': 0, 'daily_average': int(avg_daily_chars)}
+            else:
+                result['characters'] = {'projection': 0, 'daily_average': int(avg_daily_chars)}
+            
+            # Project games by target date
+            if config.games_target_date:
+                try:
+                    target_date = datetime.datetime.strptime(config.games_target_date, '%Y-%m-%d').date()
+                    days_until_target = (target_date - today).days
+                    projected_games = int(current_games + (avg_daily_games * days_until_target))
+                    result['games'] = {
+                        'projection': projected_games,
+                        'daily_average': round(avg_daily_games, 2),
+                        'target_date': config.games_target_date,
+                        'target': config.games_target,
+                        'current': current_games
+                    }
+                except ValueError:
+                    result['games'] = {'projection': 0, 'daily_average': round(avg_daily_games, 2)}
+            else:
+                result['games'] = {'projection': 0, 'daily_average': round(avg_daily_games, 2)}
+            
+            return jsonify(result), 200
+            
+        except Exception as e:
+            logger.error(f"Error calculating goal projections: {e}")
+            return jsonify({'error': 'Failed to calculate projections'}), 500
 
     @app.route('/api/import-exstatic', methods=['POST'])
     def api_import_exstatic():
@@ -1111,4 +1607,127 @@ def register_database_api_routes(app):
             
         except Exception as e:
             logger.error(f"Error in ExStatic import: {e}")
-            return jsonify({'error': f'Import failed: {str(e)}'}), 500
+            return jsonify({'error': f'Import failed: {str(e)}'}), 500
+
+    @app.route('/api/kanji-sorting-configs', methods=['GET'])
+    def api_kanji_sorting_configs():
+        """
+        List available kanji sorting configuration JSON files.
+        Returns metadata for each available sorting option.
+        """
+        try:
+            # Get the kanji_grid directory path
+            template_dir = Path(__file__).parent / 'templates' / 'components' / 'kanji_grid'
+            
+            if not template_dir.exists():
+                logger.warning(f"Kanji grid directory does not exist: {template_dir}")
+                return jsonify({'configs': []}), 200
+            
+            configs = []
+            
+            # Scan for JSON files in the directory
+            for json_file in template_dir.glob('*.json'):
+                try:
+                    with open(json_file, 'r', encoding='utf-8') as f:
+                        data = json.load(f)
+                        
+                        # Extract metadata from JSON
+                        configs.append({
+                            'filename': json_file.name,
+                            'name': data.get('name', json_file.stem),
+                            'version': data.get('version', 1),
+                            'lang': data.get('lang', 'ja'),
+                            'source': data.get('source', ''),
+                            'group_count': len(data.get('groups', []))
+                        })
+                except json.JSONDecodeError as e:
+                    logger.warning(f"Failed to parse {json_file.name}: {e}")
+                    continue
+                except Exception as e:
+                    logger.warning(f"Error reading {json_file.name}: {e}")
+                    continue
+            
+            # Sort by name for consistency
+            configs.sort(key=lambda x: x['name'])
+            
+            return jsonify({'configs': configs}), 200
+            
+        except Exception as e:
+            logger.error(f"Error fetching kanji sorting configs: {e}")
+            return jsonify({'error': 'Failed to fetch sorting configurations'}), 500
+
+    @app.route('/api/kanji-sorting-config/<filename>', methods=['GET'])
+    def api_kanji_sorting_config(filename):
+        """
+        Get a specific kanji sorting configuration file.
+        Returns the full JSON configuration.
+        """
+        try:
+            # Sanitize filename to prevent path traversal
+            if '..' in filename or '/' in filename or '\\' in filename:
+                return jsonify({'error': 'Invalid filename'}), 400
+            
+            if not filename.endswith('.json'):
+                filename += '.json'
+            
+            # Get the kanji_grid directory path
+            template_dir = Path(__file__).parent / 'templates' / 'components' / 'kanji_grid'
+            config_file = template_dir / filename
+            
+            if not config_file.exists() or not config_file.is_file():
+                return jsonify({'error': 'Configuration file not found'}), 404
+            
+            # Read and return the JSON configuration
+            with open(config_file, 'r', encoding='utf-8') as f:
+                config_data = json.load(f)
+            
+            return jsonify(config_data), 200
+            
+        except json.JSONDecodeError as e:
+            logger.error(f"Failed to parse {filename}: {e}")
+            return jsonify({'error': 'Invalid JSON configuration'}), 500
+        except Exception as e:
+            logger.error(f"Error fetching config {filename}: {e}")
+            return jsonify({'error': 'Failed to fetch configuration'}), 500
+
+    @app.route('/api/debug-db', methods=['GET'])
+    def api_debug_db():
+        """Debug endpoint to check database structure and content."""
+        try:
+            # Check table structure
+            columns_info = GameLinesTable._db.fetchall("PRAGMA table_info(game_lines)")
+            table_structure = [{'name': col[1], 'type': col[2], 'notnull': col[3], 'default': col[4]} for col in columns_info]
+            
+            # Check if we have any data
+            count_result = GameLinesTable._db.fetchone("SELECT COUNT(*) FROM game_lines")
+            total_count = count_result[0] if count_result else 0
+            
+            # Try to get a sample record
+            sample_record = None
+            if total_count > 0:
+                sample_row = GameLinesTable._db.fetchone("SELECT * FROM game_lines LIMIT 1")
+                if sample_row:
+                    sample_record = {
+                        'row_length': len(sample_row),
+                        'sample_data': sample_row[:5] if len(sample_row) > 5 else sample_row  # First 5 columns only
+                    }
+            
+            # Test the model
+            model_info = {
+                'fields_count': len(GameLinesTable._fields),
+                'types_count': len(GameLinesTable._types),
+                'fields': GameLinesTable._fields,
+                'types': [str(t) for t in GameLinesTable._types]
+            }
+            
+            return jsonify({
+                'table_structure': table_structure,
+                'total_records': total_count,
+                'sample_record': sample_record,
+                'model_info': model_info
+            }), 200
+            
+        except Exception as e:
+            logger.error(f"Error in debug endpoint: {e}")
+            return jsonify({'error': f'Debug failed: {str(e)}'}), 500
+