GameSentenceMiner 2.19.16__py3-none-any.whl → 2.20.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of GameSentenceMiner might be problematic. Click here for more details.

Files changed (70) hide show
  1. GameSentenceMiner/__init__.py +39 -0
  2. GameSentenceMiner/anki.py +6 -3
  3. GameSentenceMiner/gametext.py +13 -2
  4. GameSentenceMiner/gsm.py +40 -3
  5. GameSentenceMiner/locales/en_us.json +4 -0
  6. GameSentenceMiner/locales/ja_jp.json +4 -0
  7. GameSentenceMiner/locales/zh_cn.json +4 -0
  8. GameSentenceMiner/obs.py +4 -1
  9. GameSentenceMiner/owocr/owocr/ocr.py +304 -134
  10. GameSentenceMiner/owocr/owocr/run.py +1 -1
  11. GameSentenceMiner/ui/anki_confirmation.py +4 -2
  12. GameSentenceMiner/ui/config_gui.py +12 -0
  13. GameSentenceMiner/util/configuration.py +6 -2
  14. GameSentenceMiner/util/cron/__init__.py +12 -0
  15. GameSentenceMiner/util/cron/daily_rollup.py +613 -0
  16. GameSentenceMiner/util/cron/jiten_update.py +397 -0
  17. GameSentenceMiner/util/cron/populate_games.py +154 -0
  18. GameSentenceMiner/util/cron/run_crons.py +148 -0
  19. GameSentenceMiner/util/cron/setup_populate_games_cron.py +118 -0
  20. GameSentenceMiner/util/cron_table.py +334 -0
  21. GameSentenceMiner/util/db.py +236 -49
  22. GameSentenceMiner/util/ffmpeg.py +23 -4
  23. GameSentenceMiner/util/games_table.py +340 -93
  24. GameSentenceMiner/util/jiten_api_client.py +188 -0
  25. GameSentenceMiner/util/stats_rollup_table.py +216 -0
  26. GameSentenceMiner/web/anki_api_endpoints.py +438 -220
  27. GameSentenceMiner/web/database_api.py +955 -1259
  28. GameSentenceMiner/web/jiten_database_api.py +1015 -0
  29. GameSentenceMiner/web/rollup_stats.py +672 -0
  30. GameSentenceMiner/web/static/css/dashboard-shared.css +75 -13
  31. GameSentenceMiner/web/static/css/overview.css +604 -47
  32. GameSentenceMiner/web/static/css/search.css +226 -0
  33. GameSentenceMiner/web/static/css/shared.css +762 -0
  34. GameSentenceMiner/web/static/css/stats.css +221 -0
  35. GameSentenceMiner/web/static/js/components/bar-chart.js +339 -0
  36. GameSentenceMiner/web/static/js/database-bulk-operations.js +320 -0
  37. GameSentenceMiner/web/static/js/database-game-data.js +390 -0
  38. GameSentenceMiner/web/static/js/database-game-operations.js +213 -0
  39. GameSentenceMiner/web/static/js/database-helpers.js +44 -0
  40. GameSentenceMiner/web/static/js/database-jiten-integration.js +750 -0
  41. GameSentenceMiner/web/static/js/database-popups.js +89 -0
  42. GameSentenceMiner/web/static/js/database-tabs.js +64 -0
  43. GameSentenceMiner/web/static/js/database-text-management.js +371 -0
  44. GameSentenceMiner/web/static/js/database.js +86 -718
  45. GameSentenceMiner/web/static/js/goals.js +79 -18
  46. GameSentenceMiner/web/static/js/heatmap.js +29 -23
  47. GameSentenceMiner/web/static/js/overview.js +1205 -339
  48. GameSentenceMiner/web/static/js/regex-patterns.js +100 -0
  49. GameSentenceMiner/web/static/js/search.js +215 -18
  50. GameSentenceMiner/web/static/js/shared.js +193 -39
  51. GameSentenceMiner/web/static/js/stats.js +1536 -179
  52. GameSentenceMiner/web/stats.py +1142 -269
  53. GameSentenceMiner/web/stats_api.py +2104 -0
  54. GameSentenceMiner/web/templates/anki_stats.html +4 -18
  55. GameSentenceMiner/web/templates/components/date-range.html +118 -3
  56. GameSentenceMiner/web/templates/components/html-head.html +40 -6
  57. GameSentenceMiner/web/templates/components/js-config.html +8 -8
  58. GameSentenceMiner/web/templates/components/regex-input.html +160 -0
  59. GameSentenceMiner/web/templates/database.html +564 -117
  60. GameSentenceMiner/web/templates/goals.html +41 -5
  61. GameSentenceMiner/web/templates/overview.html +159 -129
  62. GameSentenceMiner/web/templates/search.html +78 -9
  63. GameSentenceMiner/web/templates/stats.html +159 -5
  64. GameSentenceMiner/web/texthooking_page.py +280 -111
  65. {gamesentenceminer-2.19.16.dist-info → gamesentenceminer-2.20.0.dist-info}/METADATA +43 -2
  66. {gamesentenceminer-2.19.16.dist-info → gamesentenceminer-2.20.0.dist-info}/RECORD +70 -47
  67. {gamesentenceminer-2.19.16.dist-info → gamesentenceminer-2.20.0.dist-info}/WHEEL +0 -0
  68. {gamesentenceminer-2.19.16.dist-info → gamesentenceminer-2.20.0.dist-info}/entry_points.txt +0 -0
  69. {gamesentenceminer-2.19.16.dist-info → gamesentenceminer-2.20.0.dist-info}/licenses/LICENSE +0 -0
  70. {gamesentenceminer-2.19.16.dist-info → gamesentenceminer-2.20.0.dist-info}/top_level.txt +0 -0
@@ -0,0 +1,39 @@
1
+ import os
2
+
3
+ # Remove environment variables that could interfere with managed Python instance
4
+ # This prevents conflicts with user's system Python installations and configurations
5
+
6
+ # Tk/Tcl libraries
7
+ os.environ.pop('TCL_LIBRARY', None)
8
+ os.environ.pop('TK_LIBRARY', None)
9
+
10
+ # Python-specific paths that could cause module conflicts
11
+ os.environ.pop('PYTHONPATH', None)
12
+ os.environ.pop('PYTHONHOME', None)
13
+ os.environ.pop('PYTHONSTARTUP', None)
14
+ os.environ.pop('PYTHONUSERBASE', None)
15
+
16
+ # Virtual environment variables
17
+ os.environ.pop('VIRTUAL_ENV', None)
18
+ os.environ.pop('CONDA_PREFIX', None)
19
+ os.environ.pop('CONDA_DEFAULT_ENV', None)
20
+ os.environ.pop('CONDA_PYTHON_EXE', None)
21
+
22
+ # Python version managers
23
+ os.environ.pop('PYENV_ROOT', None)
24
+ os.environ.pop('PYENV_VERSION', None)
25
+ os.environ.pop('PYENV_SHELL', None)
26
+
27
+ # Poetry package manager
28
+ os.environ.pop('POETRY_ACTIVE', None)
29
+ os.environ.pop('POETRY_HOME', None)
30
+
31
+ # Pip configuration that could override behavior
32
+ os.environ.pop('PIP_CONFIG_FILE', None)
33
+ os.environ.pop('PIP_REQUIRE_VIRTUALENV', None)
34
+
35
+ # Prevent user site-packages from being loaded
36
+ # os.environ['PYTHONNOUSERSITE'] = '1'
37
+
38
+ # Isolate from system installations
39
+ os.environ['PYTHONIOENCODING'] = 'utf-8'
GameSentenceMiner/anki.py CHANGED
@@ -238,12 +238,15 @@ def update_anki_card(last_note: 'AnkiCard', note=None, audio_path='', video_path
238
238
  config_app: 'ConfigApp' = gsm_state.config_app
239
239
  sentence = note['fields'].get(config.anki.sentence_field, last_note.get_field(config.anki.sentence_field))
240
240
 
241
- use_voice, sentence, translation, new_ss_path, add_nsfw_tag = config_app.show_anki_confirmation_dialog(
241
+ use_voice, sentence, translation, new_ss_path, add_nsfw_tag, new_audio_path = config_app.show_anki_confirmation_dialog(
242
242
  tango, sentence, assets.screenshot_path, assets.audio_path if update_audio_flag else None, translation, ss_time
243
243
  )
244
244
  note['fields'][config.anki.sentence_field] = sentence
245
245
  note['fields'][config.ai.anki_field] = translation
246
246
  assets.screenshot_path = new_ss_path or assets.screenshot_path
247
+ # Update audio path if TTS was generated in the dialog
248
+ if new_audio_path:
249
+ assets.audio_path = new_audio_path
247
250
 
248
251
  # Add NSFW tag if checkbox was selected
249
252
  if add_nsfw_tag:
@@ -588,13 +591,13 @@ def update_card_from_same_sentence(last_card, lines, game_line):
588
591
  time_elapsed += 0.5
589
592
  if time_elapsed > 15:
590
593
  logger.info(f"Timed out waiting for Anki update for card {last_card.noteId}, retrieving new audio")
591
- queue_card_for_processing(last_card, lines)
594
+ queue_card_for_processing(last_card, lines, game_line)
592
595
  return
593
596
  anki_result = anki_results[game_line.id]
594
597
 
595
598
  if anki_result.word == last_card.get_field(get_config().anki.word_field):
596
599
  logger.info(f"Same word detected, attempting to get new audio for card {last_card.noteId}")
597
- queue_card_for_processing(last_card, lines)
600
+ queue_card_for_processing(last_card, lines, game_line)
598
601
  return
599
602
 
600
603
  if anki_result.success:
@@ -12,6 +12,7 @@ from rapidfuzz import fuzz
12
12
 
13
13
  from GameSentenceMiner.util.configuration import get_config, gsm_status, logger, gsm_state, is_dev
14
14
  from GameSentenceMiner.util.db import GameLinesTable
15
+ from GameSentenceMiner.util.games_table import GamesTable
15
16
  from GameSentenceMiner.util.gsm_utils import do_text_replacements, TEXT_REPLACEMENTS_FILE, run_new_thread
16
17
  from GameSentenceMiner import obs
17
18
  from GameSentenceMiner.util.gsm_utils import add_srt_line
@@ -262,8 +263,18 @@ async def add_line_to_text_log(line, line_time=None):
262
263
  if get_overlay_processor().ready:
263
264
  asyncio.create_task(get_overlay_processor().find_box_and_send_to_overlay(current_line_after_regex))
264
265
  add_srt_line(line_time, new_line)
265
- if 'nostatspls' not in new_line.scene.lower():
266
- GameLinesTable.add_line(new_line)
266
+
267
+ # Link the game_line to the games table, but skip if 'nostatspls' in scene
268
+ game_line = get_text_log()[-1]
269
+ if 'nostatspls' not in game_line.scene.lower():
270
+ if game_line.scene:
271
+ # Get or create the game record
272
+ game = GamesTable.get_or_create_by_name(game_line.scene)
273
+ # Add the line with the game_id
274
+ GameLinesTable.add_line(game_line, game_id=game.id)
275
+ else:
276
+ # Fallback if no scene is set
277
+ GameLinesTable.add_line(game_line)
267
278
 
268
279
  def reset_line_hotkey_pressed():
269
280
  global current_line_time
GameSentenceMiner/gsm.py CHANGED
@@ -13,6 +13,8 @@ def handle_error_in_initialization(e):
13
13
  sys.exit(1)
14
14
 
15
15
  Icon = None
16
+ Menu = None
17
+ MenuItem = None
16
18
 
17
19
  try:
18
20
  import GameSentenceMiner.util.configuration
@@ -34,14 +36,12 @@ try:
34
36
  import signal
35
37
  import datetime
36
38
  from subprocess import Popen
37
- os.environ.pop('TCL_LIBRARY', None)
38
39
 
39
40
  import keyboard
40
41
  import ttkbootstrap as ttk
41
42
  from PIL import Image
42
43
  try:
43
- pass
44
- # from pystray import Icon, Menu, MenuItem
44
+ from pystray import Icon, Menu, MenuItem
45
45
  except Exception:
46
46
  logger.warning("pystray not installed correctly, tray icon will not work.")
47
47
  from watchdog.events import FileSystemEventHandler
@@ -562,6 +562,15 @@ gsm_tray = GSMTray()
562
562
  def close_obs():
563
563
  obs.disconnect_from_obs()
564
564
  if obs.obs_process_pid:
565
+ if is_linux() or is_mac():
566
+ try:
567
+ os.kill(obs.obs_process_pid, signal.SIGTERM)
568
+ print(f"OBS (PID {obs.obs_process_pid}) has been terminated.")
569
+ if os.path.exists(obs.OBS_PID_FILE):
570
+ os.remove(obs.OBS_PID_FILE)
571
+ except Exception as e:
572
+ print(f"Error terminating OBS: {e}")
573
+ return
565
574
  try:
566
575
  subprocess.run(["taskkill", "/PID", str(obs.obs_process_pid),
567
576
  "/F"], check=True, capture_output=True, text=True)
@@ -704,6 +713,34 @@ def initialize(reloading=False):
704
713
  if is_mac():
705
714
  if shutil.which("ffmpeg") is None:
706
715
  os.environ["PATH"] += os.pathsep + "/opt/homebrew/bin"
716
+
717
+ # Check for due cron jobs on startup
718
+ try:
719
+ from GameSentenceMiner.util.cron.run_crons import run_due_crons
720
+ logger.info("Checking for due cron jobs...")
721
+ run_due_crons()
722
+ except Exception as e:
723
+ logger.warning(f"Failed to check cron jobs on startup: {e}")
724
+
725
+ # Check if rollup table needs initial population (version upgrade migration)
726
+ try:
727
+ from GameSentenceMiner.util.stats_rollup_table import StatsRollupTable
728
+ from GameSentenceMiner.util.db import GameLinesTable
729
+ from GameSentenceMiner.util.cron.daily_rollup import run_daily_rollup
730
+
731
+ # Check if we have game lines but no rollup data
732
+ first_rollup = StatsRollupTable.get_first_date()
733
+ has_game_lines = GameLinesTable._db.fetchone(
734
+ f"SELECT COUNT(*) FROM {GameLinesTable._table}"
735
+ )[0] > 0
736
+
737
+ if has_game_lines and not first_rollup:
738
+ logger.info("Detected existing data without rollup table - running initial rollup generation...")
739
+ logger.info("This is a one-time migration for version upgrades. Please wait...")
740
+ rollup_result = run_daily_rollup()
741
+ logger.info(f"Initial rollup complete: processed {rollup_result.get('processed', 0)} dates")
742
+ except Exception as e:
743
+ logger.warning(f"Failed to check/populate rollup table on startup: {e}")
707
744
  if get_config().obs.open_obs:
708
745
  obs_process = obs.start_obs()
709
746
  # obs.connect_to_obs(start_replay=True)
@@ -428,6 +428,10 @@
428
428
  "label": "Close OBS:",
429
429
  "tooltip": "Close OBS when the GSM closes."
430
430
  },
431
+ "obs_path": {
432
+ "label": "OBS Path:",
433
+ "tooltip": "Path to the OBS executable. Leave blank to use the default path."
434
+ },
431
435
  "host": {
432
436
  "label": "Host:",
433
437
  "tooltip": "Host address for the OBS WebSocket server."
@@ -427,6 +427,10 @@
427
427
  "label": "OBSを閉じる:",
428
428
  "tooltip": "GSM終了時にOBSを閉じます。"
429
429
  },
430
+ "obs_path": {
431
+ "label": "OBSパス:",
432
+ "tooltip": "OBS実行ファイルのパス。空欄の場合はデフォルトパスを使用します。"
433
+ },
430
434
  "host": {
431
435
  "label": "ホスト:",
432
436
  "tooltip": "OBS WebSocketサーバーのホストアドレス。"
@@ -428,6 +428,10 @@
428
428
  "label": "关闭 OBS:",
429
429
  "tooltip": "GSM 关闭时关闭 OBS。"
430
430
  },
431
+ "obs_path": {
432
+ "label": "OBS 路径:",
433
+ "tooltip": "OBS 可执行文件的路径。留空使用默认路径。"
434
+ },
431
435
  "host": {
432
436
  "label": "主机:",
433
437
  "tooltip": "OBS WebSocket 服务器的主机地址。"
GameSentenceMiner/obs.py CHANGED
@@ -37,7 +37,7 @@ def get_queued_gui_errors():
37
37
  connection_pool: 'OBSConnectionPool' = None
38
38
  event_client: obs.EventClient = None
39
39
  obs_process_pid = None
40
- OBS_PID_FILE = os.path.join(configuration.get_app_directory(), 'obs-studio', 'obs_pid.txt')
40
+ OBS_PID_FILE = os.path.join(configuration.get_app_directory(), 'obs_pid.txt')
41
41
  obs_connection_manager = None
42
42
  logging.getLogger("obsws_python").setLevel(logging.CRITICAL)
43
43
  connecting = False
@@ -257,6 +257,9 @@ def get_base_obs_dir():
257
257
  return os.path.join(configuration.get_app_directory(), 'obs-studio')
258
258
 
259
259
  def get_obs_path():
260
+ config = get_config()
261
+ if config.obs.obs_path:
262
+ return config.obs.obs_path
260
263
  return os.path.join(configuration.get_app_directory(), 'obs-studio/bin/64bit/obs64.exe')
261
264
 
262
265
  def is_process_running(pid):