GameSentenceMiner 2.19.16__py3-none-any.whl → 2.20.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of GameSentenceMiner might be problematic. Click here for more details.
- GameSentenceMiner/__init__.py +39 -0
- GameSentenceMiner/anki.py +6 -3
- GameSentenceMiner/gametext.py +13 -2
- GameSentenceMiner/gsm.py +40 -3
- GameSentenceMiner/locales/en_us.json +4 -0
- GameSentenceMiner/locales/ja_jp.json +4 -0
- GameSentenceMiner/locales/zh_cn.json +4 -0
- GameSentenceMiner/obs.py +4 -1
- GameSentenceMiner/owocr/owocr/ocr.py +304 -134
- GameSentenceMiner/owocr/owocr/run.py +1 -1
- GameSentenceMiner/ui/anki_confirmation.py +4 -2
- GameSentenceMiner/ui/config_gui.py +12 -0
- GameSentenceMiner/util/configuration.py +6 -2
- GameSentenceMiner/util/cron/__init__.py +12 -0
- GameSentenceMiner/util/cron/daily_rollup.py +613 -0
- GameSentenceMiner/util/cron/jiten_update.py +397 -0
- GameSentenceMiner/util/cron/populate_games.py +154 -0
- GameSentenceMiner/util/cron/run_crons.py +148 -0
- GameSentenceMiner/util/cron/setup_populate_games_cron.py +118 -0
- GameSentenceMiner/util/cron_table.py +334 -0
- GameSentenceMiner/util/db.py +236 -49
- GameSentenceMiner/util/ffmpeg.py +23 -4
- GameSentenceMiner/util/games_table.py +340 -93
- GameSentenceMiner/util/jiten_api_client.py +188 -0
- GameSentenceMiner/util/stats_rollup_table.py +216 -0
- GameSentenceMiner/web/anki_api_endpoints.py +438 -220
- GameSentenceMiner/web/database_api.py +955 -1259
- GameSentenceMiner/web/jiten_database_api.py +1015 -0
- GameSentenceMiner/web/rollup_stats.py +672 -0
- GameSentenceMiner/web/static/css/dashboard-shared.css +75 -13
- GameSentenceMiner/web/static/css/overview.css +604 -47
- GameSentenceMiner/web/static/css/search.css +226 -0
- GameSentenceMiner/web/static/css/shared.css +762 -0
- GameSentenceMiner/web/static/css/stats.css +221 -0
- GameSentenceMiner/web/static/js/components/bar-chart.js +339 -0
- GameSentenceMiner/web/static/js/database-bulk-operations.js +320 -0
- GameSentenceMiner/web/static/js/database-game-data.js +390 -0
- GameSentenceMiner/web/static/js/database-game-operations.js +213 -0
- GameSentenceMiner/web/static/js/database-helpers.js +44 -0
- GameSentenceMiner/web/static/js/database-jiten-integration.js +750 -0
- GameSentenceMiner/web/static/js/database-popups.js +89 -0
- GameSentenceMiner/web/static/js/database-tabs.js +64 -0
- GameSentenceMiner/web/static/js/database-text-management.js +371 -0
- GameSentenceMiner/web/static/js/database.js +86 -718
- GameSentenceMiner/web/static/js/goals.js +79 -18
- GameSentenceMiner/web/static/js/heatmap.js +29 -23
- GameSentenceMiner/web/static/js/overview.js +1205 -339
- GameSentenceMiner/web/static/js/regex-patterns.js +100 -0
- GameSentenceMiner/web/static/js/search.js +215 -18
- GameSentenceMiner/web/static/js/shared.js +193 -39
- GameSentenceMiner/web/static/js/stats.js +1536 -179
- GameSentenceMiner/web/stats.py +1142 -269
- GameSentenceMiner/web/stats_api.py +2104 -0
- GameSentenceMiner/web/templates/anki_stats.html +4 -18
- GameSentenceMiner/web/templates/components/date-range.html +118 -3
- GameSentenceMiner/web/templates/components/html-head.html +40 -6
- GameSentenceMiner/web/templates/components/js-config.html +8 -8
- GameSentenceMiner/web/templates/components/regex-input.html +160 -0
- GameSentenceMiner/web/templates/database.html +564 -117
- GameSentenceMiner/web/templates/goals.html +41 -5
- GameSentenceMiner/web/templates/overview.html +159 -129
- GameSentenceMiner/web/templates/search.html +78 -9
- GameSentenceMiner/web/templates/stats.html +159 -5
- GameSentenceMiner/web/texthooking_page.py +280 -111
- {gamesentenceminer-2.19.16.dist-info → gamesentenceminer-2.20.0.dist-info}/METADATA +43 -2
- {gamesentenceminer-2.19.16.dist-info → gamesentenceminer-2.20.0.dist-info}/RECORD +70 -47
- {gamesentenceminer-2.19.16.dist-info → gamesentenceminer-2.20.0.dist-info}/WHEEL +0 -0
- {gamesentenceminer-2.19.16.dist-info → gamesentenceminer-2.20.0.dist-info}/entry_points.txt +0 -0
- {gamesentenceminer-2.19.16.dist-info → gamesentenceminer-2.20.0.dist-info}/licenses/LICENSE +0 -0
- {gamesentenceminer-2.19.16.dist-info → gamesentenceminer-2.20.0.dist-info}/top_level.txt +0 -0
GameSentenceMiner/__init__.py
CHANGED
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
import os
|
|
2
|
+
|
|
3
|
+
# Remove environment variables that could interfere with managed Python instance
|
|
4
|
+
# This prevents conflicts with user's system Python installations and configurations
|
|
5
|
+
|
|
6
|
+
# Tk/Tcl libraries
|
|
7
|
+
os.environ.pop('TCL_LIBRARY', None)
|
|
8
|
+
os.environ.pop('TK_LIBRARY', None)
|
|
9
|
+
|
|
10
|
+
# Python-specific paths that could cause module conflicts
|
|
11
|
+
os.environ.pop('PYTHONPATH', None)
|
|
12
|
+
os.environ.pop('PYTHONHOME', None)
|
|
13
|
+
os.environ.pop('PYTHONSTARTUP', None)
|
|
14
|
+
os.environ.pop('PYTHONUSERBASE', None)
|
|
15
|
+
|
|
16
|
+
# Virtual environment variables
|
|
17
|
+
os.environ.pop('VIRTUAL_ENV', None)
|
|
18
|
+
os.environ.pop('CONDA_PREFIX', None)
|
|
19
|
+
os.environ.pop('CONDA_DEFAULT_ENV', None)
|
|
20
|
+
os.environ.pop('CONDA_PYTHON_EXE', None)
|
|
21
|
+
|
|
22
|
+
# Python version managers
|
|
23
|
+
os.environ.pop('PYENV_ROOT', None)
|
|
24
|
+
os.environ.pop('PYENV_VERSION', None)
|
|
25
|
+
os.environ.pop('PYENV_SHELL', None)
|
|
26
|
+
|
|
27
|
+
# Poetry package manager
|
|
28
|
+
os.environ.pop('POETRY_ACTIVE', None)
|
|
29
|
+
os.environ.pop('POETRY_HOME', None)
|
|
30
|
+
|
|
31
|
+
# Pip configuration that could override behavior
|
|
32
|
+
os.environ.pop('PIP_CONFIG_FILE', None)
|
|
33
|
+
os.environ.pop('PIP_REQUIRE_VIRTUALENV', None)
|
|
34
|
+
|
|
35
|
+
# Prevent user site-packages from being loaded
|
|
36
|
+
# os.environ['PYTHONNOUSERSITE'] = '1'
|
|
37
|
+
|
|
38
|
+
# Isolate from system installations
|
|
39
|
+
os.environ['PYTHONIOENCODING'] = 'utf-8'
|
GameSentenceMiner/anki.py
CHANGED
|
@@ -238,12 +238,15 @@ def update_anki_card(last_note: 'AnkiCard', note=None, audio_path='', video_path
|
|
|
238
238
|
config_app: 'ConfigApp' = gsm_state.config_app
|
|
239
239
|
sentence = note['fields'].get(config.anki.sentence_field, last_note.get_field(config.anki.sentence_field))
|
|
240
240
|
|
|
241
|
-
use_voice, sentence, translation, new_ss_path, add_nsfw_tag = config_app.show_anki_confirmation_dialog(
|
|
241
|
+
use_voice, sentence, translation, new_ss_path, add_nsfw_tag, new_audio_path = config_app.show_anki_confirmation_dialog(
|
|
242
242
|
tango, sentence, assets.screenshot_path, assets.audio_path if update_audio_flag else None, translation, ss_time
|
|
243
243
|
)
|
|
244
244
|
note['fields'][config.anki.sentence_field] = sentence
|
|
245
245
|
note['fields'][config.ai.anki_field] = translation
|
|
246
246
|
assets.screenshot_path = new_ss_path or assets.screenshot_path
|
|
247
|
+
# Update audio path if TTS was generated in the dialog
|
|
248
|
+
if new_audio_path:
|
|
249
|
+
assets.audio_path = new_audio_path
|
|
247
250
|
|
|
248
251
|
# Add NSFW tag if checkbox was selected
|
|
249
252
|
if add_nsfw_tag:
|
|
@@ -588,13 +591,13 @@ def update_card_from_same_sentence(last_card, lines, game_line):
|
|
|
588
591
|
time_elapsed += 0.5
|
|
589
592
|
if time_elapsed > 15:
|
|
590
593
|
logger.info(f"Timed out waiting for Anki update for card {last_card.noteId}, retrieving new audio")
|
|
591
|
-
queue_card_for_processing(last_card, lines)
|
|
594
|
+
queue_card_for_processing(last_card, lines, game_line)
|
|
592
595
|
return
|
|
593
596
|
anki_result = anki_results[game_line.id]
|
|
594
597
|
|
|
595
598
|
if anki_result.word == last_card.get_field(get_config().anki.word_field):
|
|
596
599
|
logger.info(f"Same word detected, attempting to get new audio for card {last_card.noteId}")
|
|
597
|
-
queue_card_for_processing(last_card, lines)
|
|
600
|
+
queue_card_for_processing(last_card, lines, game_line)
|
|
598
601
|
return
|
|
599
602
|
|
|
600
603
|
if anki_result.success:
|
GameSentenceMiner/gametext.py
CHANGED
|
@@ -12,6 +12,7 @@ from rapidfuzz import fuzz
|
|
|
12
12
|
|
|
13
13
|
from GameSentenceMiner.util.configuration import get_config, gsm_status, logger, gsm_state, is_dev
|
|
14
14
|
from GameSentenceMiner.util.db import GameLinesTable
|
|
15
|
+
from GameSentenceMiner.util.games_table import GamesTable
|
|
15
16
|
from GameSentenceMiner.util.gsm_utils import do_text_replacements, TEXT_REPLACEMENTS_FILE, run_new_thread
|
|
16
17
|
from GameSentenceMiner import obs
|
|
17
18
|
from GameSentenceMiner.util.gsm_utils import add_srt_line
|
|
@@ -262,8 +263,18 @@ async def add_line_to_text_log(line, line_time=None):
|
|
|
262
263
|
if get_overlay_processor().ready:
|
|
263
264
|
asyncio.create_task(get_overlay_processor().find_box_and_send_to_overlay(current_line_after_regex))
|
|
264
265
|
add_srt_line(line_time, new_line)
|
|
265
|
-
|
|
266
|
-
|
|
266
|
+
|
|
267
|
+
# Link the game_line to the games table, but skip if 'nostatspls' in scene
|
|
268
|
+
game_line = get_text_log()[-1]
|
|
269
|
+
if 'nostatspls' not in game_line.scene.lower():
|
|
270
|
+
if game_line.scene:
|
|
271
|
+
# Get or create the game record
|
|
272
|
+
game = GamesTable.get_or_create_by_name(game_line.scene)
|
|
273
|
+
# Add the line with the game_id
|
|
274
|
+
GameLinesTable.add_line(game_line, game_id=game.id)
|
|
275
|
+
else:
|
|
276
|
+
# Fallback if no scene is set
|
|
277
|
+
GameLinesTable.add_line(game_line)
|
|
267
278
|
|
|
268
279
|
def reset_line_hotkey_pressed():
|
|
269
280
|
global current_line_time
|
GameSentenceMiner/gsm.py
CHANGED
|
@@ -13,6 +13,8 @@ def handle_error_in_initialization(e):
|
|
|
13
13
|
sys.exit(1)
|
|
14
14
|
|
|
15
15
|
Icon = None
|
|
16
|
+
Menu = None
|
|
17
|
+
MenuItem = None
|
|
16
18
|
|
|
17
19
|
try:
|
|
18
20
|
import GameSentenceMiner.util.configuration
|
|
@@ -34,14 +36,12 @@ try:
|
|
|
34
36
|
import signal
|
|
35
37
|
import datetime
|
|
36
38
|
from subprocess import Popen
|
|
37
|
-
os.environ.pop('TCL_LIBRARY', None)
|
|
38
39
|
|
|
39
40
|
import keyboard
|
|
40
41
|
import ttkbootstrap as ttk
|
|
41
42
|
from PIL import Image
|
|
42
43
|
try:
|
|
43
|
-
|
|
44
|
-
# from pystray import Icon, Menu, MenuItem
|
|
44
|
+
from pystray import Icon, Menu, MenuItem
|
|
45
45
|
except Exception:
|
|
46
46
|
logger.warning("pystray not installed correctly, tray icon will not work.")
|
|
47
47
|
from watchdog.events import FileSystemEventHandler
|
|
@@ -562,6 +562,15 @@ gsm_tray = GSMTray()
|
|
|
562
562
|
def close_obs():
|
|
563
563
|
obs.disconnect_from_obs()
|
|
564
564
|
if obs.obs_process_pid:
|
|
565
|
+
if is_linux() or is_mac():
|
|
566
|
+
try:
|
|
567
|
+
os.kill(obs.obs_process_pid, signal.SIGTERM)
|
|
568
|
+
print(f"OBS (PID {obs.obs_process_pid}) has been terminated.")
|
|
569
|
+
if os.path.exists(obs.OBS_PID_FILE):
|
|
570
|
+
os.remove(obs.OBS_PID_FILE)
|
|
571
|
+
except Exception as e:
|
|
572
|
+
print(f"Error terminating OBS: {e}")
|
|
573
|
+
return
|
|
565
574
|
try:
|
|
566
575
|
subprocess.run(["taskkill", "/PID", str(obs.obs_process_pid),
|
|
567
576
|
"/F"], check=True, capture_output=True, text=True)
|
|
@@ -704,6 +713,34 @@ def initialize(reloading=False):
|
|
|
704
713
|
if is_mac():
|
|
705
714
|
if shutil.which("ffmpeg") is None:
|
|
706
715
|
os.environ["PATH"] += os.pathsep + "/opt/homebrew/bin"
|
|
716
|
+
|
|
717
|
+
# Check for due cron jobs on startup
|
|
718
|
+
try:
|
|
719
|
+
from GameSentenceMiner.util.cron.run_crons import run_due_crons
|
|
720
|
+
logger.info("Checking for due cron jobs...")
|
|
721
|
+
run_due_crons()
|
|
722
|
+
except Exception as e:
|
|
723
|
+
logger.warning(f"Failed to check cron jobs on startup: {e}")
|
|
724
|
+
|
|
725
|
+
# Check if rollup table needs initial population (version upgrade migration)
|
|
726
|
+
try:
|
|
727
|
+
from GameSentenceMiner.util.stats_rollup_table import StatsRollupTable
|
|
728
|
+
from GameSentenceMiner.util.db import GameLinesTable
|
|
729
|
+
from GameSentenceMiner.util.cron.daily_rollup import run_daily_rollup
|
|
730
|
+
|
|
731
|
+
# Check if we have game lines but no rollup data
|
|
732
|
+
first_rollup = StatsRollupTable.get_first_date()
|
|
733
|
+
has_game_lines = GameLinesTable._db.fetchone(
|
|
734
|
+
f"SELECT COUNT(*) FROM {GameLinesTable._table}"
|
|
735
|
+
)[0] > 0
|
|
736
|
+
|
|
737
|
+
if has_game_lines and not first_rollup:
|
|
738
|
+
logger.info("Detected existing data without rollup table - running initial rollup generation...")
|
|
739
|
+
logger.info("This is a one-time migration for version upgrades. Please wait...")
|
|
740
|
+
rollup_result = run_daily_rollup()
|
|
741
|
+
logger.info(f"Initial rollup complete: processed {rollup_result.get('processed', 0)} dates")
|
|
742
|
+
except Exception as e:
|
|
743
|
+
logger.warning(f"Failed to check/populate rollup table on startup: {e}")
|
|
707
744
|
if get_config().obs.open_obs:
|
|
708
745
|
obs_process = obs.start_obs()
|
|
709
746
|
# obs.connect_to_obs(start_replay=True)
|
|
@@ -428,6 +428,10 @@
|
|
|
428
428
|
"label": "Close OBS:",
|
|
429
429
|
"tooltip": "Close OBS when the GSM closes."
|
|
430
430
|
},
|
|
431
|
+
"obs_path": {
|
|
432
|
+
"label": "OBS Path:",
|
|
433
|
+
"tooltip": "Path to the OBS executable. Leave blank to use the default path."
|
|
434
|
+
},
|
|
431
435
|
"host": {
|
|
432
436
|
"label": "Host:",
|
|
433
437
|
"tooltip": "Host address for the OBS WebSocket server."
|
GameSentenceMiner/obs.py
CHANGED
|
@@ -37,7 +37,7 @@ def get_queued_gui_errors():
|
|
|
37
37
|
connection_pool: 'OBSConnectionPool' = None
|
|
38
38
|
event_client: obs.EventClient = None
|
|
39
39
|
obs_process_pid = None
|
|
40
|
-
OBS_PID_FILE = os.path.join(configuration.get_app_directory(), '
|
|
40
|
+
OBS_PID_FILE = os.path.join(configuration.get_app_directory(), 'obs_pid.txt')
|
|
41
41
|
obs_connection_manager = None
|
|
42
42
|
logging.getLogger("obsws_python").setLevel(logging.CRITICAL)
|
|
43
43
|
connecting = False
|
|
@@ -257,6 +257,9 @@ def get_base_obs_dir():
|
|
|
257
257
|
return os.path.join(configuration.get_app_directory(), 'obs-studio')
|
|
258
258
|
|
|
259
259
|
def get_obs_path():
|
|
260
|
+
config = get_config()
|
|
261
|
+
if config.obs.obs_path:
|
|
262
|
+
return config.obs.obs_path
|
|
260
263
|
return os.path.join(configuration.get_app_directory(), 'obs-studio/bin/64bit/obs64.exe')
|
|
261
264
|
|
|
262
265
|
def is_process_running(pid):
|