GameSentenceMiner 2.19.16__py3-none-any.whl → 2.20.0__py3-none-any.whl

GameSentenceMiner/util/cron/jiten_update.py

@@ -0,0 +1,397 @@
+"""
+Jiten.moe Update Script for GameSentenceMiner
+
+This module provides functions to automatically update game metadata from jiten.moe,
+respecting manual overrides set by users.
+
+Usage:
+    from GameSentenceMiner.util.cron.jiten_update import update_all_jiten_games
+
+    # Update all linked games
+    result = update_all_jiten_games()
+    print(f"Updated {result['updated_games']} out of {result['total_games']} games")
+"""
+
+import time
+from typing import Optional, Dict, List
+
+from GameSentenceMiner.util.games_table import GamesTable
+from GameSentenceMiner.util.configuration import logger
+from GameSentenceMiner.util.jiten_api_client import JitenApiClient
+
+
+def fetch_jiten_data_for_game(game: GamesTable) -> Optional[Dict]:
+    """
+    Fetch fresh data from jiten.moe API for a specific game.
+
+    Args:
+        game: GamesTable object with a deck_id
+
+    Returns:
+        Dictionary with normalized jiten.moe data (snake_case keys), or None if fetch fails
+
+    Example return structure:
+        {
+            'deck_id': 123,
+            'title_original': '君と彼女と彼女の恋。',
+            'title_romaji': 'Kimi to Kanojo to Kanojo no Koi.',
+            'title_english': 'You, Me, and Her',
+            'description': 'A visual novel about...',
+            'cover_name': 'https://...',
+            'media_type': 7,  # 1=Anime, 7=Visual Novel, 2=Manga
+            'character_count': 50000,
+            'difficulty': 5,
+            'difficulty_raw': 5.2,
+            'links': [...],
+            'aliases': [...],
+            'release_date': '2013-06-28'
+        }
+    """
+    if not game.deck_id:
+        logger.debug(
+            f"Game {game.id} ({game.title_original}) has no deck_id, skipping jiten fetch"
+        )
+        return None
+
+    try:
+        logger.debug(
+            f"Fetching jiten.moe data for game: {game.title_original} (deck_id: {game.deck_id})"
+        )
+
+        # Use direct deck detail API endpoint
+        data = JitenApiClient.get_deck_detail(game.deck_id)
+
+        if not data:
+            logger.debug(f"Failed to fetch deck detail for deck_id {game.deck_id}")
+            return None
+
+        # Extract main deck data from the detail response
+        main_deck = data.get("data", {}).get("mainDeck")
+        if not main_deck:
+            logger.debug(f"No mainDeck found in response for deck_id {game.deck_id}")
+            return None
+
+        # Normalize the deck data
+        jiten_data = JitenApiClient.normalize_deck_data(main_deck)
+        logger.debug(
+            f"Successfully fetched jiten.moe data for: {jiten_data['title_original']}"
+        )
+        return jiten_data
+
+    except Exception as e:
+        logger.debug(f"Unexpected error fetching jiten data for game {game.id}: {e}")
+        return None
+
+
+def update_single_game_from_jiten(game: GamesTable, jiten_data: Dict) -> Dict:
+    """
+    Update a single game's fields from jiten.moe data, respecting manual overrides.
+    Always re-downloads cover images.
+
+    Args:
+        game: GamesTable object to update
+        jiten_data: Dictionary with jiten.moe data (from fetch_jiten_data_for_game)
+
+    Returns:
+        Dictionary with update summary:
+        {
+            'success': bool,
+            'updated_fields': List[str],
+            'skipped_fields': List[str],
+            'error': Optional[str]
+        }
+    """
+    try:
+        update_fields = {}
+        skipped_fields = []
+
+        # Ensure manual_overrides is always a list
+        manual_overrides = (
+            game.manual_overrides if game.manual_overrides is not None else []
+        )
+        if not isinstance(manual_overrides, list):
+            logger.warning(
+                f"⚠️ manual_overrides is not a list for game {game.id}: {type(manual_overrides)}"
+            )
+            manual_overrides = []
+
+        logger.debug(
+            f"Checking fields for game {game.id} (manual overrides: {manual_overrides})"
+        )
+
+        # Check each field against manual overrides
+        if "deck_id" not in manual_overrides:
+            update_fields["deck_id"] = jiten_data["deck_id"]
+        else:
+            skipped_fields.append("deck_id")
+
+        if "title_original" not in manual_overrides and jiten_data.get(
+            "title_original"
+        ):
+            update_fields["title_original"] = jiten_data["title_original"]
+        elif "title_original" in manual_overrides:
+            skipped_fields.append("title_original")
+
+        if "title_romaji" not in manual_overrides and jiten_data.get("title_romaji"):
+            update_fields["title_romaji"] = jiten_data["title_romaji"]
+        elif "title_romaji" in manual_overrides:
+            skipped_fields.append("title_romaji")
+
+        if "title_english" not in manual_overrides and jiten_data.get("title_english"):
+            update_fields["title_english"] = jiten_data["title_english"]
+        elif "title_english" in manual_overrides:
+            skipped_fields.append("title_english")
+
+        if "type" not in manual_overrides and jiten_data.get("media_type"):
+            # Map media type to string
+            media_type_map = {1: "Anime", 7: "Visual Novel", 2: "Manga"}
+            update_fields["game_type"] = media_type_map.get(
+                jiten_data["media_type"], "Unknown"
+            )
+        elif "type" in manual_overrides:
+            skipped_fields.append("type")
+
+        if "description" not in manual_overrides and jiten_data.get("description"):
+            update_fields["description"] = jiten_data["description"]
+        elif "description" in manual_overrides:
+            skipped_fields.append("description")
+
+        if (
+            "difficulty" not in manual_overrides
+            and jiten_data.get("difficulty") is not None
+        ):
+            update_fields["difficulty"] = jiten_data["difficulty"]
+        elif "difficulty" in manual_overrides:
+            skipped_fields.append("difficulty")
+
+        if (
+            "character_count" not in manual_overrides
+            and jiten_data.get("character_count") is not None
+        ):
+            update_fields["character_count"] = jiten_data["character_count"]
+        elif "character_count" in manual_overrides:
+            skipped_fields.append("character_count")
+
+        if "links" not in manual_overrides and jiten_data.get("links"):
+            update_fields["links"] = jiten_data["links"]
+        elif "links" in manual_overrides:
+            skipped_fields.append("links")
+
+        if "release_date" not in manual_overrides and jiten_data.get("release_date"):
+            update_fields["release_date"] = jiten_data["release_date"]
+        elif "release_date" in manual_overrides:
+            skipped_fields.append("release_date")
+
+        # Always re-download image if not manually overridden
+        if "image" not in manual_overrides and jiten_data.get("cover_name"):
+            image_data = JitenApiClient.download_cover_image(jiten_data["cover_name"])
+            if image_data:
+                update_fields["image"] = image_data
+                logger.debug(f"Downloaded and encoded image for game {game.id}")
+            else:
+                logger.debug(f"Failed to download image for game {game.id}")
+        elif "image" in manual_overrides:
+            skipped_fields.append("image")
+
+        # Update the game using the jiten update method (doesn't mark as manual)
+        if update_fields:
+            game.update_all_fields_from_jiten(**update_fields)
+            logger.debug(
+                f"Updated game {game.id} ({game.title_original}): {len(update_fields)} fields"
+            )
+            return {
+                "success": True,
+                "updated_fields": list(update_fields.keys()),
+                "skipped_fields": skipped_fields,
+                "error": None,
+            }
+        else:
+            logger.debug(
+                f"No fields updated for game {game.id} - all fields are manually overridden"
+            )
+            return {
+                "success": True,
+                "updated_fields": [],
+                "skipped_fields": skipped_fields,
+                "error": None,
+            }
+
+    except Exception as e:
+        logger.error(
+            f"💥 Error updating game {game.id} from jiten data: {e}", exc_info=True
+        )
+        return {
+            "success": False,
+            "updated_fields": [],
+            "skipped_fields": skipped_fields,
+            "error": str(e),
+        }
+
+
+def update_all_jiten_games() -> Dict:
+    """
+    Update all games that are linked to jiten.moe (have a deck_id).
+
+    This is the main entry point for the jiten update cron job.
+    - Continues on errors (individual game failures don't stop the process)
+    - Always re-downloads images
+    - Adds 1 second delay between games to avoid rate limiting
+
+    Returns:
+        Dictionary with summary statistics:
+        {
+            'total_games': int,           # Total games in database
+            'linked_games': int,          # Games with deck_id
+            'updated_games': int,         # Successfully updated
+            'failed_games': int,          # Failed to update
+            'skipped_games': int,         # No deck_id
+            'total_fields_updated': int,  # Total fields updated across all games
+            'details': List[Dict]         # Per-game details
+        }
+    """
+    logger.debug("Starting jiten.moe update for all linked games")
+
+    start_time = time.time()
+
+    # Get all games
+    all_games = GamesTable.all()
+    total_games = len(all_games)
+
+    # Filter for linked games (have deck_id)
+    linked_games = [game for game in all_games if game.deck_id]
+    linked_count = len(linked_games)
+    skipped_count = total_games - linked_count
+
+    logger.debug(
+        f"Found {total_games} total games, {linked_count} linked to jiten.moe, {skipped_count} unlinked"
+    )
+
+    if linked_count == 0:
+        logger.debug("No linked games found, nothing to update")
+        return {
+            "total_games": total_games,
+            "linked_games": 0,
+            "updated_games": 0,
+            "failed_games": 0,
+            "skipped_games": skipped_count,
+            "total_fields_updated": 0,
+            "details": [],
+        }
+
+    # Process each linked game
+    updated_count = 0
+    failed_count = 0
+    total_fields_updated = 0
+    details = []
+
+    for i, game in enumerate(linked_games, 1):
+        logger.debug(
+            f"Processing game {i}/{linked_count}: {game.title_original} (deck_id: {game.deck_id})"
+        )
+
+        game_detail = {
+            "game_id": game.id,
+            "title": game.title_original,
+            "deck_id": game.deck_id,
+            "success": False,
+            "updated_fields": [],
+            "skipped_fields": [],
+            "error": None,
+        }
+
+        try:
+            # Fetch jiten data
+            jiten_data = fetch_jiten_data_for_game(game)
+
+            if jiten_data is None:
+                logger.debug(f"Failed to fetch jiten data for game {game.id}, skipping")
+                failed_count += 1
+                game_detail["error"] = "Failed to fetch jiten data"
+                details.append(game_detail)
+                continue
+
+            # Update the game
+            result = update_single_game_from_jiten(game, jiten_data)
+
+            if result["success"]:
+                updated_count += 1
+                total_fields_updated += len(result["updated_fields"])
+                game_detail["success"] = True
+                game_detail["updated_fields"] = result["updated_fields"]
+                game_detail["skipped_fields"] = result["skipped_fields"]
+            else:
+                failed_count += 1
+                game_detail["error"] = result["error"]
+
+            details.append(game_detail)
+
+        except Exception as e:
+            logger.error(
+                f"💥 Unexpected error processing game {game.id}: {e}", exc_info=True
+            )
+            failed_count += 1
+            game_detail["error"] = str(e)
+            details.append(game_detail)
+
+        # Add 1 second delay between games (except after the last one)
+        if i < linked_count:
+            logger.debug(f"Waiting 1 second before next game...")
+            time.sleep(1)
+
+    elapsed_time = time.time() - start_time
+
+    # Log summary
+    logger.debug("Jiten.moe update completed")
+    logger.debug(f"Summary:")
+    logger.debug(f"   - Total games: {total_games}")
+    logger.debug(f"   - Linked games: {linked_count}")
+    logger.debug(f"   - Successfully updated: {updated_count}")
+    logger.debug(f"   - Failed: {failed_count}")
+    logger.debug(f"   - Skipped (no deck_id): {skipped_count}")
+    logger.debug(f"   - Total fields updated: {total_fields_updated}")
+    logger.debug(f"   - Time elapsed: {elapsed_time:.2f} seconds")
+
+    return {
+        "total_games": total_games,
+        "linked_games": linked_count,
+        "updated_games": updated_count,
+        "failed_games": failed_count,
+        "skipped_games": skipped_count,
+        "total_fields_updated": total_fields_updated,
+        "elapsed_time": elapsed_time,
+        "details": details,
+    }
+
+
+# Example usage for testing
+if __name__ == "__main__":
+    # Run the update
+    result = update_all_jiten_games()
+
+    # Print summary
+    print("\n" + "=" * 80)
+    print("JITEN UPDATE SUMMARY")
+    print("=" * 80)
+    print(f"Total games: {result['total_games']}")
+    print(f"Linked games: {result['linked_games']}")
+    print(f"Successfully updated: {result['updated_games']}")
+    print(f"Failed: {result['failed_games']}")
+    print(f"Skipped (no deck_id): {result['skipped_games']}")
+    print(f"Total fields updated: {result['total_fields_updated']}")
+    print(f"Time elapsed: {result['elapsed_time']:.2f} seconds")
+    print("=" * 80)
+
+    # Print per-game details
+    if result["details"]:
+        print("\nPER-GAME DETAILS:")
+        print("-" * 80)
+        for detail in result["details"]:
+            status = "✅" if detail["success"] else "❌"
+            print(f"{status} {detail['title']} (deck_id: {detail['deck_id']})")
+            if detail["updated_fields"]:
+                print(f"   Updated: {', '.join(detail['updated_fields'])}")
+            if detail["skipped_fields"]:
+                print(f"   Skipped: {', '.join(detail['skipped_fields'])}")
+            if detail["error"]:
+                print(f"   Error: {detail['error']}")
+        print("-" * 80)

GameSentenceMiner/util/cron/populate_games.py

@@ -0,0 +1,154 @@
+"""
+One-time Game Population Cron Job for GameSentenceMiner
+
+This module provides a one-time cron job that auto-creates game records from game_lines.
+This ensures the games table is populated before the daily rollup runs, so that
+game_activity_data and games_played_ids can be properly populated in the rollup.
+
+This job should run once and then mark itself as complete.
+
+Usage:
+    from GameSentenceMiner.util.cron.populate_games import populate_games_table
+    
+    # Run the one-time population
+    result = populate_games_table()
+    print(f"Created {result['created']} games, linked {result['linked_lines']} lines")
+"""
+
+from GameSentenceMiner.util.db import GameLinesTable
+from GameSentenceMiner.util.games_table import GamesTable
+from GameSentenceMiner.util.configuration import logger
+
+
+def populate_games_table():
+    """
+    Auto-create game records for any game_lines that don't have corresponding games.
+    This is a one-time operation that ensures the games table is populated.
+    
+    Returns:
+        Dictionary with execution summary:
+        {
+            'success': bool,
+            'created': int,  # Number of new games created
+            'linked_lines': int,  # Number of game_lines linked to games
+            'errors': int,
+            'error_message': str or None
+        }
+    """
+    logger.info("Starting one-time game population from game_lines")
+    
+    created_count = 0
+    linked_lines_count = 0
+    errors = 0
+    
+    try:
+        # Get all distinct game names from game_lines
+        game_names_from_lines = GameLinesTable._db.fetchall(
+            f"SELECT DISTINCT game_name FROM {GameLinesTable._table} "
+            f"WHERE game_name IS NOT NULL AND game_name != ''"
+        )
+        
+        if not game_names_from_lines:
+            logger.info("No game names found in game_lines table")
+            return {
+                'success': True,
+                'created': 0,
+                'linked_lines': 0,
+                'errors': 0,
+                'error_message': None
+            }
+        
+        logger.info(f"Found {len(game_names_from_lines)} distinct game names in game_lines")
+        
+        # Get existing game titles to avoid duplicates
+        existing_games_rows = GamesTable._db.fetchall(
+            f"SELECT title_original FROM {GamesTable._table}"
+        )
+        existing_titles = {row[0] for row in existing_games_rows}
+        
+        logger.info(f"Found {len(existing_titles)} existing games in games table")
+        
+        # Auto-create games for game_lines that don't have corresponding games
+        for row in game_names_from_lines:
+            game_name = row[0]
+            
+            # Skip if game already exists
+            if game_name in existing_titles:
+                logger.debug(f"Game '{game_name}' already exists, skipping")
+                continue
+            
+            try:
+                # Use get_or_create_by_name which checks for existing mappings
+                # and reuses them instead of creating duplicates
+                game = GamesTable.get_or_create_by_name(game_name)
+                
+                # Link any orphaned game_lines to this game
+                GameLinesTable._db.execute(
+                    f"UPDATE {GameLinesTable._table} SET game_id = ? "
+                    f"WHERE game_name = ? AND (game_id IS NULL OR game_id = '')",
+                    (game.id, game_name),
+                    commit=True,
+                )
+                
+                # Count how many lines were linked
+                linked_count = GameLinesTable._db.fetchone(
+                    f"SELECT COUNT(*) FROM {GameLinesTable._table} WHERE game_id = ?",
+                    (game.id,),
+                )
+                lines_linked = linked_count[0] if linked_count else 0
+                
+                logger.info(
+                    f"Created game '{game_name}' (id={game.id}) and linked {lines_linked} lines"
+                )
+                
+                created_count += 1
+                linked_lines_count += lines_linked
+                existing_titles.add(game_name)
+                
+            except Exception as e:
+                logger.error(f"Error creating game for '{game_name}': {e}", exc_info=True)
+                errors += 1
+                continue
+        
+        logger.info(
+            f"Game population completed: created {created_count} games, "
+            f"linked {linked_lines_count} lines, {errors} errors"
+        )
+        
+        return {
+            'success': True,
+            'created': created_count,
+            'linked_lines': linked_lines_count,
+            'errors': errors,
+            'error_message': None
+        }
+        
+    except Exception as e:
+        error_msg = str(e)
+        logger.error(f"Fatal error in populate_games_table: {error_msg}", exc_info=True)
+        
+        return {
+            'success': False,
+            'created': created_count,
+            'linked_lines': linked_lines_count,
+            'errors': errors + 1,
+            'error_message': error_msg
+        }
+
+
+# Example usage for testing
+if __name__ == '__main__':
+    # Run the one-time population
+    result = populate_games_table()
+    
+    # Print summary
+    print("\n" + "=" * 80)
+    print("GAME POPULATION SUMMARY")
+    print("=" * 80)
+    print(f"Success: {'Yes' if result['success'] else 'No'}")
+    print(f"Games created: {result['created']}")
+    print(f"Lines linked: {result['linked_lines']}")
+    print(f"Errors: {result['errors']}")
+    if result['error_message']:
+        print(f"Error: {result['error_message']}")
+    print("=" * 80)

GameSentenceMiner/util/cron/run_crons.py

@@ -0,0 +1,148 @@
+"""
+Cron Job Runner for GameSentenceMiner
+
+This script checks for due cron jobs and executes them.
+Should be called periodically (e.g., every hour) by an external scheduler.
+
+Usage:
+    python -m GameSentenceMiner.util.cron.run_crons
+"""
+
+from GameSentenceMiner.util.cron_table import CronTable
+from GameSentenceMiner.util.configuration import logger
+
+
+def run_due_crons():
+    """
+    Check for and execute all due cron jobs.
+    
+    Returns:
+        Dictionary with execution summary
+    """
+    logger.info("Checking for due cron jobs...")
+    
+    # Get all cron jobs that need to run
+    due_crons = CronTable.get_due_crons()
+    
+    if not due_crons:
+        logger.info("No cron jobs are due to run at this time")
+        return {
+            'total_checked': 0,
+            'executed': 0,
+            'failed': 0,
+            'details': []
+        }
+    
+    logger.info(f"📋 Found {len(due_crons)} cron job(s) due to run")
+    
+    executed_count = 0
+    failed_count = 0
+    details = []
+    
+    for cron in due_crons:
+        logger.info(f"Executing cron job: {cron.name}")
+        logger.info(f"Description: {cron.description}")
+        
+        detail = {
+            'name': cron.name,
+            'description': cron.description,
+            'success': False,
+            'error': None
+        }
+        
+        try:
+            # Execute populate_games BEFORE daily_stats_rollup to ensure games table is populated
+            if cron.name == 'populate_games':
+                from GameSentenceMiner.util.cron.populate_games import populate_games_table
+                result = populate_games_table()
+                
+                # Mark as successfully run (even if there were some errors, as long as it completed)
+                CronTable.just_ran(cron.id)
+                executed_count += 1
+                detail['success'] = True
+                detail['result'] = result
+                
+                logger.info(f"Successfully executed {cron.name}")
+                logger.info(f"Created: {result['created']} games, Linked: {result['linked_lines']} lines, Errors: {result['errors']}")
+                
+            # Execute the appropriate function based on cron name
+            elif cron.name == 'jiten_sync':
+                from GameSentenceMiner.util.cron.jiten_update import update_all_jiten_games
+                result = update_all_jiten_games()
+                
+                # Mark as successfully run
+                CronTable.just_ran(cron.id)
+                executed_count += 1
+                detail['success'] = True
+                detail['result'] = result
+                
+                logger.info(f"Successfully executed {cron.name}")
+                logger.info(f"Updated: {result['updated_games']}/{result['linked_games']} games")
+                
+            elif cron.name == 'daily_stats_rollup':
+                from GameSentenceMiner.util.cron.daily_rollup import run_daily_rollup
+                result = run_daily_rollup()
+                
+                # Mark as successfully run
+                CronTable.just_ran(cron.id)
+                executed_count += 1
+                detail['success'] = True
+                detail['result'] = result
+                
+                logger.info(f"Successfully executed {cron.name}")
+                logger.info(f"Processed: {result['processed']} dates, Overwritten: {result['overwritten']}, Errors: {result['errors']}")
+                
+            else:
+                logger.error(f"⚠️ Unknown cron job: {cron.name}")
+                detail['error'] = f"Unknown cron job: {cron.name}"
+                failed_count += 1
+                
+        except Exception as e:
+            logger.error(f"Failed to execute {cron.name}: {e}", exc_info=True)
+            detail['error'] = str(e)
+            failed_count += 1
+        
+        details.append(detail)
+    logger.info("Cron job check completed")
+    logger.info(f"Total checked: {len(due_crons)}")
+    logger.info(f"Successfully executed: {executed_count}")
+    logger.info(f"Failed: {failed_count}")
+
+    return {
+        'total_checked': len(due_crons),
+        'executed': executed_count,
+        'failed': failed_count,
+        'details': details
+    }
+
+
+# for me to manually check
+if __name__ == '__main__':
+    # Run the cron checker
+    result = run_due_crons()
+    
+    # Print summary
+    print("\n" + "=" * 80)
+    print("CRON EXECUTION SUMMARY")
+    print("=" * 80)
+    print(f"Total cron jobs checked: {result['total_checked']}")
+    print(f"Successfully executed: {result['executed']}")
+    print(f"Failed: {result['failed']}")
+    print("=" * 80)
+    
+    # Print details
+    if result['details']:
+        print("\nDETAILS:")
+        print("-" * 80)
+        for detail in result['details']:
+            status = "✅" if detail['success'] else "❌"
+            print(f"{status} {detail['name']}: {detail['description']}")
+            if detail.get('error'):
+                print(f"   Error: {detail['error']}")
+            elif detail.get('result'):
+                res = detail['result']
+                if 'updated_games' in res:
+                    print(f"   Updated {res['updated_games']}/{res['linked_games']} games")
+                elif 'processed' in res:
+                    print(f"   Processed {res['processed']} dates, Overwritten {res['overwritten']}, Errors {res['errors']}")
+        print("-" * 80)