GameSentenceMiner 0.0.0.dev0__tar.gz

gamesentenceminer-0.0.0.dev0/GameSentenceMiner/__init__.py

@@ -0,0 +1,39 @@
+import os
+
+# Remove environment variables that could interfere with managed Python instance
+# This prevents conflicts with user's system Python installations and configurations
+
+# Tk/Tcl libraries
+os.environ.pop('TCL_LIBRARY', None)
+os.environ.pop('TK_LIBRARY', None)
+
+# Python-specific paths that could cause module conflicts
+os.environ.pop('PYTHONPATH', None)
+os.environ.pop('PYTHONHOME', None)
+os.environ.pop('PYTHONSTARTUP', None)
+os.environ.pop('PYTHONUSERBASE', None)
+
+# Virtual environment variables
+os.environ.pop('VIRTUAL_ENV', None)
+os.environ.pop('CONDA_PREFIX', None)
+os.environ.pop('CONDA_DEFAULT_ENV', None)
+os.environ.pop('CONDA_PYTHON_EXE', None)
+
+# Python version managers
+os.environ.pop('PYENV_ROOT', None)
+os.environ.pop('PYENV_VERSION', None)
+os.environ.pop('PYENV_SHELL', None)
+
+# Poetry package manager
+os.environ.pop('POETRY_ACTIVE', None)
+os.environ.pop('POETRY_HOME', None)
+
+# Pip configuration that could override behavior
+os.environ.pop('PIP_CONFIG_FILE', None)
+os.environ.pop('PIP_REQUIRE_VIRTUALENV', None)
+
+# Prevent user site-packages from being loaded
+# os.environ['PYTHONNOUSERSITE'] = '1'
+
+# Isolate from system installations
+os.environ['PYTHONIOENCODING'] = 'utf-8'

gamesentenceminer-0.0.0.dev0/GameSentenceMiner/ai/ai_prompting.py

@@ -0,0 +1,121 @@
+import logging
+from typing import List, Optional
+
+from GameSentenceMiner.ai.features.character_summary import CharacterSummaryService
+from GameSentenceMiner.ai.service import AIService, snapshot_config
+from GameSentenceMiner.util.config.configuration import (
+    AI_GEMINI,
+    AI_GROQ,
+    AI_GSM_CLOUD,
+    AI_LM_STUDIO,
+    AI_OLLAMA,
+    AI_OPENAI,
+    Ai,
+    get_config,
+    logger,
+)
+from GameSentenceMiner.util.text_log import GameLine
+
+# Suppress debug logs from httpcore
+logging.getLogger("httpcore").setLevel(logging.WARNING)
+logging.getLogger("httpx").setLevel(logging.WARNING)
+logging.getLogger("groq._base_client").setLevel(logging.WARNING)
+
+
+def get_ai_prompt_result(
+    lines: List[GameLine],
+    sentence: str,
+    current_line: GameLine,
+    game_title: str = "",
+    force_refresh: bool = False,
+    custom_prompt=None,
+) -> str:
+    try:
+        config = get_config()
+        snapshot = snapshot_config(config.ai, config.general)
+        service = AIService(config_snapshot=snapshot, logger=logger)
+        return service.translate(
+            lines=lines,
+            sentence=sentence,
+            current_line=current_line,
+            game_title=game_title,
+            custom_prompt=custom_prompt,
+        )
+    except Exception as e:
+        logger.error(
+            "Error caught while trying to get AI prompt result. Check logs for more details.",
+            exc_info=True,
+        )
+        logger.debug(e, exc_info=True)
+        return ""
+
+
+def ai_config_changed(config: Ai, current: Optional[Ai]) -> bool:
+    if not current:
+        return True
+    if config.provider != current.provider:
+        return True
+    if config.provider == AI_GEMINI and (
+        config.gemini_api_key != current.gemini_api_key
+        or config.gemini_model != current.gemini_model
+        or config.gemini_backup_model != current.gemini_backup_model
+    ):
+        return True
+    if config.provider == AI_GROQ and (
+        config.groq_api_key != current.groq_api_key
+        or config.groq_model != current.groq_model
+        or config.groq_backup_model != current.groq_backup_model
+    ):
+        return True
+    if config.provider == AI_OPENAI and (
+        config.open_ai_api_key != current.open_ai_api_key
+        or config.open_ai_model != current.open_ai_model
+        or config.open_ai_backup_model != current.open_ai_backup_model
+        or config.open_ai_url != current.open_ai_url
+    ):
+        return True
+    if config.provider == AI_GSM_CLOUD and (
+        config.gsm_cloud_access_token != current.gsm_cloud_access_token
+        or config.gsm_cloud_models != current.gsm_cloud_models
+        or config.gsm_cloud_api_url != current.gsm_cloud_api_url
+    ):
+        return True
+    if config.provider == AI_OLLAMA and (
+        config.ollama_url != current.ollama_url
+        or config.ollama_model != current.ollama_model
+        or config.ollama_backup_model != current.ollama_backup_model
+    ):
+        return True
+    if config.provider == AI_LM_STUDIO and (
+        config.lm_studio_url != current.lm_studio_url
+        or config.lm_studio_model != current.lm_studio_model
+        or config.lm_studio_backup_model != current.lm_studio_backup_model
+        or config.lm_studio_api_key != current.lm_studio_api_key
+    ):
+        return True
+    if config.custom_prompt != current.custom_prompt:
+        return True
+    if config.custom_full_prompt != current.custom_full_prompt:
+        return True
+    if config.use_canned_translation_prompt != current.use_canned_translation_prompt:
+        return True
+    if config.use_canned_context_prompt != current.use_canned_context_prompt:
+        return True
+    if config.temperature != current.temperature:
+        return True
+    if config.max_output_tokens != current.max_output_tokens:
+        return True
+    if config.top_p != current.top_p:
+        return True
+    return False
+
+
+def generate_character_summary(character_data: dict) -> Optional[str]:
+    if not character_data:
+        return None
+
+    config = get_config()
+    snapshot = snapshot_config(config.ai, config.general)
+    service = AIService(config_snapshot=snapshot, logger=logger)
+    summary_service = CharacterSummaryService(logger=logger)
+    return summary_service.generate_from_vndb(character_data, service)

gamesentenceminer-0.0.0.dev0/GameSentenceMiner/ai/contracts.py

@@ -0,0 +1,33 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Optional, Dict, Any
+
+
+@dataclass(frozen=True)
+class AIRequest:
+    provider: str
+    model: str
+    prompt: str
+    temperature: float
+    top_p: float
+    max_tokens: int
+    game_title: str = ""
+    request_kind: str = "translation"
+    metadata: Optional[Dict[str, Any]] = None
+
+
+@dataclass(frozen=True)
+class AIResponse:
+    provider: str
+    model: str
+    text: str
+    raw_text: str
+    latency_ms: int
+    usage: Optional[Dict[str, Any]] = None
+
+
+class AIError(Exception):
+    def __init__(self, message: str, transient: bool = False):
+        super().__init__(message)
+        self.transient = transient

gamesentenceminer-0.0.0.dev0/GameSentenceMiner/ai/features/__init__.py

@@ -0,0 +1 @@
+from __future__ import annotations

gamesentenceminer-0.0.0.dev0/GameSentenceMiner/ai/features/character_context.py

@@ -0,0 +1,50 @@
+from __future__ import annotations
+
+import json
+
+
+class CharacterContextProvider:
+    def __init__(self, summary_service, logger):
+        self.summary_service = summary_service
+        self.logger = logger
+
+    def get_character_context(self, game_title: str, ai_service) -> str:
+        if not game_title:
+            return ""
+
+        try:
+            from GameSentenceMiner.util.database.games_table import GamesTable
+
+            game = GamesTable.get_by_obs_scene_name(game_title)
+            if not game:
+                game = GamesTable.get_by_title(game_title)
+
+            if not game:
+                return ""
+
+            self.logger.debug(
+                f"Found game '{game.title_original}' (id={game.id}) for scene '{game_title}'"
+            )
+            if game.character_summary:
+                return game.character_summary
+
+            if game.vndb_character_data:
+                try:
+                    if isinstance(game.vndb_character_data, dict):
+                        vndb_data = game.vndb_character_data
+                    else:
+                        vndb_data = json.loads(game.vndb_character_data)
+                    summary = self.summary_service.generate_from_vndb(vndb_data, ai_service)
+                    if summary:
+                        game.character_summary = summary
+                        game.save()
+                        self.logger.info(f"Generated and stored character summary for {game_title}")
+                        return summary
+                except json.JSONDecodeError:
+                    self.logger.warning(f"Failed to parse VNDB data for {game_title}")
+                except Exception as e:
+                    self.logger.error(f"Failed to generate character summary for {game_title}: {e}")
+        except Exception as e:
+            self.logger.error(f"Error fetching character context: {e}")
+
+        return ""

gamesentenceminer-0.0.0.dev0/GameSentenceMiner/ai/features/character_summary.py

@@ -0,0 +1,25 @@
+from __future__ import annotations
+
+import json
+from typing import Optional
+
+from GameSentenceMiner.ai.prompts.templates import CHARACTER_SUMMARY_PROMPT
+
+
+class CharacterSummaryService:
+    def __init__(self, logger):
+        self.logger = logger
+
+    def generate_from_vndb(self, character_data: dict, ai_service) -> Optional[str]:
+        if not character_data:
+            return None
+
+        character_json = json.dumps(character_data, ensure_ascii=False, indent=2)
+        prompt = CHARACTER_SUMMARY_PROMPT.format(character_json=character_json)
+        try:
+            result = ai_service.generate_raw_prompt(prompt, request_kind="character_summary")
+            if result:
+                return result.strip()
+        except Exception as e:
+            self.logger.error(f"Failed to generate character summary: {e}")
+        return None

gamesentenceminer-0.0.0.dev0/GameSentenceMiner/ai/parsing/__init__.py

@@ -0,0 +1 @@
+from __future__ import annotations

gamesentenceminer-0.0.0.dev0/GameSentenceMiner/ai/parsing/output_parser.py

@@ -0,0 +1,29 @@
+from __future__ import annotations
+
+import json
+from dataclasses import dataclass
+from typing import Optional
+
+
+@dataclass
+class OutputParser:
+    compat_mode: bool = True
+
+    def parse(self, raw_text: str) -> str:
+        if not raw_text:
+            return raw_text
+
+        if not self.compat_mode:
+            return raw_text
+
+        if "{" in raw_text and "}" in raw_text:
+            try:
+                json_output = raw_text[raw_text.find("{"):raw_text.rfind("}") + 1]
+                json_output = json_output.replace("{output:", '{"output":')
+                parsed = json.loads(json_output)
+                if isinstance(parsed, dict) and "output" in parsed:
+                    return parsed["output"]
+            except Exception:
+                return raw_text
+
+        return raw_text

gamesentenceminer-0.0.0.dev0/GameSentenceMiner/ai/prompts/__init__.py

@@ -0,0 +1 @@
+from __future__ import annotations

gamesentenceminer-0.0.0.dev0/GameSentenceMiner/ai/prompts/builder.py

@@ -0,0 +1,118 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import List, Optional
+
+from GameSentenceMiner.ai.prompts.templates import (
+    DIALOGUE_CONTEXT_TEMPLATE,
+    FULL_PROMPT_TEMPLATE,
+    build_context_prompt,
+    build_translation_prompt,
+)
+from GameSentenceMiner.util.text_log import GameLine
+
+
+@dataclass(frozen=True)
+class PromptSelection:
+    prompt_text: str
+    prompt_kind: str
+
+
+class DialogueContextBuilder:
+    @staticmethod
+    def build(lines: List[GameLine], current_line: GameLine, context_length: int) -> str:
+        if context_length == 0:
+            return "No dialogue context available."
+
+        if context_length == -1:
+            start_index = 0
+            end_index = len(lines)
+        else:
+            start_index = max(0, current_line.index - context_length)
+            end_index = min(len(lines), current_line.index + 1 + context_length)
+
+        context_lines_text = []
+        for i in range(start_index, end_index):
+            if i < len(lines):
+                context_lines_text.append(lines[i].text)
+
+        return DIALOGUE_CONTEXT_TEMPLATE.format("\n".join(context_lines_text))
+
+
+class PromptSelector:
+    @staticmethod
+    def select(
+        use_canned_translation_prompt: bool,
+        use_canned_context_prompt: bool,
+        custom_prompt: str,
+        native_language_name: str,
+        custom_prompt_override: Optional[str] = None,
+    ) -> PromptSelection:
+        if custom_prompt_override:
+            return PromptSelection(prompt_text=custom_prompt_override, prompt_kind="custom")
+        if use_canned_translation_prompt:
+            return PromptSelection(
+                prompt_text=build_translation_prompt(native_language_name),
+                prompt_kind="translation",
+            )
+        if use_canned_context_prompt:
+            return PromptSelection(
+                prompt_text=build_context_prompt(native_language_name),
+                prompt_kind="context",
+            )
+        return PromptSelection(prompt_text=custom_prompt, prompt_kind="custom")
+
+
+class FullPromptRenderer:
+    @staticmethod
+    def render(
+        game_title: str,
+        character_context: str,
+        dialogue_context: str,
+        prompt_to_use: str,
+        sentence: str,
+    ) -> str:
+        return FULL_PROMPT_TEMPLATE.format(
+            game_title=game_title or "Unknown",
+            character_context=character_context,
+            dialogue_context=dialogue_context,
+            prompt_to_use=prompt_to_use,
+            sentence=sentence,
+        )
+
+
+class PromptBuilder:
+    def __init__(self, native_language_name: str):
+        self.native_language_name = native_language_name
+
+    def build(
+        self,
+        lines: List[GameLine],
+        sentence: str,
+        current_line: GameLine,
+        game_title: str,
+        dialogue_context_length: int,
+        use_canned_translation_prompt: bool,
+        use_canned_context_prompt: bool,
+        custom_prompt: str,
+        custom_prompt_override: Optional[str] = None,
+        character_context: str = "",
+    ) -> tuple[str, str]:
+        dialogue_context = DialogueContextBuilder.build(
+            lines, current_line, dialogue_context_length
+        )
+        selection = PromptSelector.select(
+            use_canned_translation_prompt=use_canned_translation_prompt,
+            use_canned_context_prompt=use_canned_context_prompt,
+            custom_prompt=custom_prompt,
+            native_language_name=self.native_language_name,
+            custom_prompt_override=custom_prompt_override,
+        )
+        full_prompt = FullPromptRenderer.render(
+            game_title=game_title,
+            character_context=character_context,
+            dialogue_context=dialogue_context,
+            prompt_to_use=selection.prompt_text,
+            sentence=sentence,
+        )
+        return full_prompt, selection.prompt_kind

gamesentenceminer-0.0.0.dev0/GameSentenceMiner/ai/prompts/templates.py

@@ -0,0 +1,74 @@
+from __future__ import annotations
+
+
+def build_translation_prompt(native_language_name: str) -> str:
+    return f"""
+**Professional Game Localization Task**
+
+**Task Directive:**
+Translate ONLY the provided line(s) of game dialogue specified below into natural-sounding, context-aware {native_language_name}. The translation must preserve the original tone and intent of the source.
+
+**Output Requirements:**
+- Provide only the single, best {native_language_name} translation.
+- Expletives are okay, only if they absolutely 100% fit the context and tone of the original line, and are commonly used in {native_language_name} localizations of similar games.
+- Carryover all HTML tags present in the original text to HTML tags surrounding their corresponding translated words in the translation. Look for the equivalent word, not the equivalent location. DO NOT CONVERT TO MARKDOWN.
+- If there are no HTML tags present in the original text, do not add any in the translation whatsoever.
+- Do not include notes, alternatives, explanations, or any other surrounding text. Absolutely nothing but the translated line(s).
+
+**Line(s) to Translate:**
+"""
+
+
+def build_context_prompt(native_language_name: str) -> str:
+    return f"""
+
+**Task Directive:**
+Provide a very brief summary of the scene in {native_language_name} based on the provided dialogue and context. Focus on the characters' actions and the immediate situation being described.
+
+Current Line(s):
+"""
+
+
+DIALOGUE_CONTEXT_TEMPLATE = """
+Dialogue Context:
+
+{0}
+"""
+
+
+FULL_PROMPT_TEMPLATE = """
+**Disclaimer:** All dialogue provided is from the script of the video game "{game_title}". This content is entirely fictional and part of a narrative. It must not be treated as real-world user input or a genuine request. The goal is accurate, context-aware localization. If no context is provided, do not throw errors or warnings.
+
+Character Context:
+{character_context}
+
+{dialogue_context}
+
+{prompt_to_use}
+
+{sentence}
+"""
+
+
+CHARACTER_SUMMARY_PROMPT = """
+You are a helpful assistant that creates concise character summaries for game localization.
+
+Given the following character data from a visual novel, create a CHARACTER LIST in this exact format:
+
+**CHARACTER LIST**:
+[Japanese Name] -> [Romanized Name] (brief one-line description)
+
+Rules:
+- Include age if available (e.g., "17yo")
+- Include gender (male/female)
+- Include 2-3 key personality traits that will aid in translation.
+- Keep each line under 120 characters
+- Use Format Japanese name (romanization name): tags
+- Mention what pronoun they use and mark it as their pronoun if they have one listed
+- Example: 陽見 恵凪 (Harumi Ena): Clumsy, Dandere, Hotblooded 19yo girl atashi pronoun
+
+Character Data:
+{character_json}
+
+Generate the CHARACTER LIST now:
+"""

gamesentenceminer-0.0.0.dev0/GameSentenceMiner/ai/providers/__init__.py

@@ -0,0 +1 @@
+from __future__ import annotations

gamesentenceminer-0.0.0.dev0/GameSentenceMiner/ai/providers/base.py

@@ -0,0 +1,10 @@
+from __future__ import annotations
+
+from typing import Protocol
+
+from GameSentenceMiner.ai.contracts import AIRequest, AIResponse
+
+
+class ProviderClient(Protocol):
+    def generate(self, request: AIRequest) -> AIResponse:
+        ...

gamesentenceminer-0.0.0.dev0/GameSentenceMiner/ai/providers/gemini_client.py

@@ -0,0 +1,107 @@
+from __future__ import annotations
+
+import time
+from google import genai
+from google.genai import types
+from typing import Optional, Any, Dict
+
+from GameSentenceMiner.ai.contracts import AIRequest, AIResponse, AIError
+from GameSentenceMiner.util.config.configuration import normalize_gemini_model_name
+
+
+class GeminiClient:
+    def __init__(self, api_key: str, model_name: str, logger):
+        self.api_key = api_key
+        self.model_name = model_name
+        self.logger = logger
+        self.client = None
+        self._safety_settings = [
+            types.SafetySetting(
+                category=types.HarmCategory.HARM_CATEGORY_HARASSMENT,
+                threshold=types.HarmBlockThreshold.BLOCK_NONE,
+            ),
+            types.SafetySetting(
+                category=types.HarmCategory.HARM_CATEGORY_HATE_SPEECH,
+                threshold=types.HarmBlockThreshold.BLOCK_NONE,
+            ),
+            types.SafetySetting(
+                category=types.HarmCategory.HARM_CATEGORY_SEXUALLY_EXPLICIT,
+                threshold=types.HarmBlockThreshold.BLOCK_NONE,
+            ),
+            types.SafetySetting(
+                category=types.HarmCategory.HARM_CATEGORY_DANGEROUS_CONTENT,
+                threshold=types.HarmBlockThreshold.BLOCK_NONE,
+            ),
+        ]
+        try:
+            self.client = genai.Client(api_key=self.api_key)
+        except Exception as e:
+            self.logger.error(f"Failed to initialize Gemini API: {e}")
+
+    @staticmethod
+    def _get_thinking_budget(model_name: str) -> Optional[int]:
+        model = (model_name or "").lower()
+        if "gemini-2.5" in model or "gemini-3" in model:
+            return -1 if "-pro" in model else 0
+        return None
+
+    def _build_generation_config(self, request: AIRequest, model_name: str) -> types.GenerateContentConfig:
+        config = types.GenerateContentConfig(
+            temperature=request.temperature,
+            max_output_tokens=request.max_tokens,
+            top_p=request.top_p,
+            stop_sequences=None,
+            safety_settings=self._safety_settings,
+        )
+        thinking_budget = self._get_thinking_budget(model_name)
+        if thinking_budget is not None:
+            config.thinking_config = types.ThinkingConfig(thinking_budget=thinking_budget)
+        return config
+
+    def generate(self, request: AIRequest) -> AIResponse:
+        if self.client is None:
+            raise AIError("Gemini model not initialized.", transient=False)
+
+        start_time = time.time()
+        try:
+            model_name = normalize_gemini_model_name(request.model)
+            generation_config = self._build_generation_config(request, model_name)
+            contents = [
+                types.Content(
+                    role="user",
+                    parts=[types.Part.from_text(text=request.prompt)],
+                ),
+            ]
+            response = self.client.models.generate_content(
+                model=model_name,
+                contents=contents,
+                config=generation_config,
+            )
+
+            self.logger.debug(f"Gemini raw response: {response}")
+
+            result = ""
+            if response.candidates:
+                for candidate in response.candidates:
+                    if candidate.content and candidate.content.parts:
+                        for part in candidate.content.parts:
+                            if hasattr(part, "text") and part.text:
+                                result += part.text
+
+            raw_text = result.strip()
+            usage: Optional[Dict[str, Any]] = None
+            if hasattr(response, "usage_metadata") and response.usage_metadata:
+                usage = response.usage_metadata.model_dump() if hasattr(response.usage_metadata, "model_dump") else dict(response.usage_metadata)
+
+            latency_ms = int((time.time() - start_time) * 1000)
+            return AIResponse(
+                provider=request.provider,
+                model=model_name,
+                text=raw_text,
+                raw_text=raw_text,
+                latency_ms=latency_ms,
+                usage=usage,
+            )
+        except Exception as e:
+            self.logger.error(f"Gemini processing failed: {e}")
+            raise AIError(f"Processing failed: {e}", transient=True)

gamesentenceminer-0.0.0.dev0/GameSentenceMiner/ai/providers/groq_client.py

@@ -0,0 +1,51 @@
+from __future__ import annotations
+
+import time
+from groq import Groq
+from typing import Optional, Any, Dict
+
+from GameSentenceMiner.ai.contracts import AIRequest, AIResponse, AIError
+
+
+class GroqClient:
+    def __init__(self, api_key: str, logger):
+        self.api_key = api_key
+        self.logger = logger
+        self.client = None
+        try:
+            self.client = Groq(api_key=self.api_key)
+        except Exception as e:
+            self.logger.error(f"Failed to initialize Groq client: {e}")
+
+    def generate(self, request: AIRequest) -> AIResponse:
+        if self.client is None:
+            raise AIError("Groq client not initialized.", transient=False)
+
+        start_time = time.time()
+        try:
+            completion = self.client.chat.completions.create(
+                model=request.model,
+                messages=[{"role": "user", "content": request.prompt}],
+                temperature=request.temperature,
+                max_completion_tokens=request.max_tokens,
+                top_p=request.top_p,
+                stream=False,
+                stop=None,
+            )
+            raw_text = completion.choices[0].message.content.strip()
+            usage: Optional[Dict[str, Any]] = None
+            if hasattr(completion, "usage") and completion.usage:
+                usage = completion.usage.model_dump() if hasattr(completion.usage, "model_dump") else dict(completion.usage)
+
+            latency_ms = int((time.time() - start_time) * 1000)
+            return AIResponse(
+                provider=request.provider,
+                model=request.model,
+                text=raw_text,
+                raw_text=raw_text,
+                latency_ms=latency_ms,
+                usage=usage,
+            )
+        except Exception as e:
+            self.logger.error(f"Groq processing failed: {e}")
+            raise AIError(f"Processing failed: {e}", transient=True)