GameSentenceMiner 2.17.1__tar.gz → 2.17.2__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of GameSentenceMiner might be problematic. Click here for more details.

Files changed (104) hide show
  1. {gamesentenceminer-2.17.1 → gamesentenceminer-2.17.2}/GameSentenceMiner/anki.py +25 -1
  2. {gamesentenceminer-2.17.1 → gamesentenceminer-2.17.2}/GameSentenceMiner/config_gui.py +19 -1
  3. {gamesentenceminer-2.17.1 → gamesentenceminer-2.17.2}/GameSentenceMiner/gsm.py +13 -18
  4. {gamesentenceminer-2.17.1 → gamesentenceminer-2.17.2}/GameSentenceMiner/obs.py +4 -2
  5. {gamesentenceminer-2.17.1 → gamesentenceminer-2.17.2}/GameSentenceMiner/ocr/owocr_helper.py +1 -1
  6. {gamesentenceminer-2.17.1 → gamesentenceminer-2.17.2}/GameSentenceMiner/owocr/owocr/run.py +2 -2
  7. {gamesentenceminer-2.17.1 → gamesentenceminer-2.17.2}/GameSentenceMiner/util/configuration.py +7 -5
  8. {gamesentenceminer-2.17.1 → gamesentenceminer-2.17.2}/GameSentenceMiner/util/db.py +176 -8
  9. {gamesentenceminer-2.17.1 → gamesentenceminer-2.17.2}/GameSentenceMiner/util/downloader/download_tools.py +57 -24
  10. {gamesentenceminer-2.17.1 → gamesentenceminer-2.17.2}/GameSentenceMiner/util/get_overlay_coords.py +3 -0
  11. {gamesentenceminer-2.17.1 → gamesentenceminer-2.17.2}/GameSentenceMiner/util/gsm_utils.py +0 -54
  12. {gamesentenceminer-2.17.1 → gamesentenceminer-2.17.2}/GameSentenceMiner/web/database_api.py +12 -1
  13. {gamesentenceminer-2.17.1 → gamesentenceminer-2.17.2}/GameSentenceMiner/web/gsm_websocket.py +1 -1
  14. {gamesentenceminer-2.17.1 → gamesentenceminer-2.17.2}/GameSentenceMiner/web/static/css/shared.css +20 -0
  15. {gamesentenceminer-2.17.1 → gamesentenceminer-2.17.2}/GameSentenceMiner/web/static/css/stats.css +496 -1
  16. gamesentenceminer-2.17.2/GameSentenceMiner/web/static/js/anki_stats.js +168 -0
  17. {gamesentenceminer-2.17.1 → gamesentenceminer-2.17.2}/GameSentenceMiner/web/static/js/shared.js +2 -49
  18. {gamesentenceminer-2.17.1 → gamesentenceminer-2.17.2}/GameSentenceMiner/web/static/js/stats.js +274 -39
  19. {gamesentenceminer-2.17.1 → gamesentenceminer-2.17.2}/GameSentenceMiner/web/templates/anki_stats.html +36 -0
  20. {gamesentenceminer-2.17.1 → gamesentenceminer-2.17.2}/GameSentenceMiner/web/templates/index.html +1 -1
  21. {gamesentenceminer-2.17.1 → gamesentenceminer-2.17.2}/GameSentenceMiner/web/templates/stats.html +35 -15
  22. {gamesentenceminer-2.17.1 → gamesentenceminer-2.17.2}/GameSentenceMiner/web/texthooking_page.py +31 -8
  23. {gamesentenceminer-2.17.1 → gamesentenceminer-2.17.2}/GameSentenceMiner.egg-info/PKG-INFO +1 -1
  24. {gamesentenceminer-2.17.1 → gamesentenceminer-2.17.2}/PKG-INFO +1 -1
  25. {gamesentenceminer-2.17.1 → gamesentenceminer-2.17.2}/pyproject.toml +2 -2
  26. gamesentenceminer-2.17.1/GameSentenceMiner/web/static/js/anki_stats.js +0 -84
  27. {gamesentenceminer-2.17.1 → gamesentenceminer-2.17.2}/GameSentenceMiner/__init__.py +0 -0
  28. {gamesentenceminer-2.17.1 → gamesentenceminer-2.17.2}/GameSentenceMiner/ai/__init__.py +0 -0
  29. {gamesentenceminer-2.17.1 → gamesentenceminer-2.17.2}/GameSentenceMiner/ai/ai_prompting.py +0 -0
  30. {gamesentenceminer-2.17.1 → gamesentenceminer-2.17.2}/GameSentenceMiner/assets/__init__.py +0 -0
  31. {gamesentenceminer-2.17.1 → gamesentenceminer-2.17.2}/GameSentenceMiner/assets/icon.png +0 -0
  32. {gamesentenceminer-2.17.1 → gamesentenceminer-2.17.2}/GameSentenceMiner/assets/icon128.png +0 -0
  33. {gamesentenceminer-2.17.1 → gamesentenceminer-2.17.2}/GameSentenceMiner/assets/icon256.png +0 -0
  34. {gamesentenceminer-2.17.1 → gamesentenceminer-2.17.2}/GameSentenceMiner/assets/icon32.png +0 -0
  35. {gamesentenceminer-2.17.1 → gamesentenceminer-2.17.2}/GameSentenceMiner/assets/icon512.png +0 -0
  36. {gamesentenceminer-2.17.1 → gamesentenceminer-2.17.2}/GameSentenceMiner/assets/icon64.png +0 -0
  37. {gamesentenceminer-2.17.1 → gamesentenceminer-2.17.2}/GameSentenceMiner/assets/pickaxe.png +0 -0
  38. {gamesentenceminer-2.17.1 → gamesentenceminer-2.17.2}/GameSentenceMiner/gametext.py +0 -0
  39. {gamesentenceminer-2.17.1 → gamesentenceminer-2.17.2}/GameSentenceMiner/locales/en_us.json +0 -0
  40. {gamesentenceminer-2.17.1 → gamesentenceminer-2.17.2}/GameSentenceMiner/locales/ja_jp.json +0 -0
  41. {gamesentenceminer-2.17.1 → gamesentenceminer-2.17.2}/GameSentenceMiner/locales/zh_cn.json +0 -0
  42. {gamesentenceminer-2.17.1 → gamesentenceminer-2.17.2}/GameSentenceMiner/ocr/__init__.py +0 -0
  43. {gamesentenceminer-2.17.1 → gamesentenceminer-2.17.2}/GameSentenceMiner/ocr/gsm_ocr_config.py +0 -0
  44. {gamesentenceminer-2.17.1 → gamesentenceminer-2.17.2}/GameSentenceMiner/ocr/ocrconfig.py +0 -0
  45. {gamesentenceminer-2.17.1 → gamesentenceminer-2.17.2}/GameSentenceMiner/ocr/owocr_area_selector.py +0 -0
  46. {gamesentenceminer-2.17.1 → gamesentenceminer-2.17.2}/GameSentenceMiner/ocr/ss_picker.py +0 -0
  47. {gamesentenceminer-2.17.1 → gamesentenceminer-2.17.2}/GameSentenceMiner/owocr/owocr/__init__.py +0 -0
  48. {gamesentenceminer-2.17.1 → gamesentenceminer-2.17.2}/GameSentenceMiner/owocr/owocr/__main__.py +0 -0
  49. {gamesentenceminer-2.17.1 → gamesentenceminer-2.17.2}/GameSentenceMiner/owocr/owocr/config.py +0 -0
  50. {gamesentenceminer-2.17.1 → gamesentenceminer-2.17.2}/GameSentenceMiner/owocr/owocr/lens_betterproto.py +0 -0
  51. {gamesentenceminer-2.17.1 → gamesentenceminer-2.17.2}/GameSentenceMiner/owocr/owocr/ocr.py +0 -0
  52. {gamesentenceminer-2.17.1 → gamesentenceminer-2.17.2}/GameSentenceMiner/owocr/owocr/screen_coordinate_picker.py +0 -0
  53. {gamesentenceminer-2.17.1 → gamesentenceminer-2.17.2}/GameSentenceMiner/tools/__init__.py +0 -0
  54. {gamesentenceminer-2.17.1 → gamesentenceminer-2.17.2}/GameSentenceMiner/tools/audio_offset_selector.py +0 -0
  55. {gamesentenceminer-2.17.1 → gamesentenceminer-2.17.2}/GameSentenceMiner/tools/furigana_filter_preview.py +0 -0
  56. {gamesentenceminer-2.17.1 → gamesentenceminer-2.17.2}/GameSentenceMiner/tools/ss_selector.py +0 -0
  57. {gamesentenceminer-2.17.1 → gamesentenceminer-2.17.2}/GameSentenceMiner/tools/window_transparency.py +0 -0
  58. {gamesentenceminer-2.17.1 → gamesentenceminer-2.17.2}/GameSentenceMiner/util/__init__.py +0 -0
  59. {gamesentenceminer-2.17.1 → gamesentenceminer-2.17.2}/GameSentenceMiner/util/communication/__init__.py +0 -0
  60. {gamesentenceminer-2.17.1 → gamesentenceminer-2.17.2}/GameSentenceMiner/util/communication/send.py +0 -0
  61. {gamesentenceminer-2.17.1 → gamesentenceminer-2.17.2}/GameSentenceMiner/util/communication/websocket.py +0 -0
  62. {gamesentenceminer-2.17.1 → gamesentenceminer-2.17.2}/GameSentenceMiner/util/downloader/Untitled_json.py +0 -0
  63. {gamesentenceminer-2.17.1 → gamesentenceminer-2.17.2}/GameSentenceMiner/util/downloader/__init__.py +0 -0
  64. {gamesentenceminer-2.17.1 → gamesentenceminer-2.17.2}/GameSentenceMiner/util/downloader/oneocr_dl.py +0 -0
  65. {gamesentenceminer-2.17.1 → gamesentenceminer-2.17.2}/GameSentenceMiner/util/electron_config.py +0 -0
  66. {gamesentenceminer-2.17.1 → gamesentenceminer-2.17.2}/GameSentenceMiner/util/ffmpeg.py +0 -0
  67. {gamesentenceminer-2.17.1 → gamesentenceminer-2.17.2}/GameSentenceMiner/util/model.py +0 -0
  68. {gamesentenceminer-2.17.1 → gamesentenceminer-2.17.2}/GameSentenceMiner/util/notification.py +0 -0
  69. {gamesentenceminer-2.17.1 → gamesentenceminer-2.17.2}/GameSentenceMiner/util/text_log.py +0 -0
  70. {gamesentenceminer-2.17.1 → gamesentenceminer-2.17.2}/GameSentenceMiner/util/win10toast/__init__.py +0 -0
  71. {gamesentenceminer-2.17.1 → gamesentenceminer-2.17.2}/GameSentenceMiner/util/win10toast/__main__.py +0 -0
  72. {gamesentenceminer-2.17.1 → gamesentenceminer-2.17.2}/GameSentenceMiner/vad.py +0 -0
  73. {gamesentenceminer-2.17.1 → gamesentenceminer-2.17.2}/GameSentenceMiner/web/__init__.py +0 -0
  74. {gamesentenceminer-2.17.1 → gamesentenceminer-2.17.2}/GameSentenceMiner/web/events.py +0 -0
  75. {gamesentenceminer-2.17.1 → gamesentenceminer-2.17.2}/GameSentenceMiner/web/service.py +0 -0
  76. {gamesentenceminer-2.17.1 → gamesentenceminer-2.17.2}/GameSentenceMiner/web/static/__init__.py +0 -0
  77. {gamesentenceminer-2.17.1 → gamesentenceminer-2.17.2}/GameSentenceMiner/web/static/apple-touch-icon.png +0 -0
  78. {gamesentenceminer-2.17.1 → gamesentenceminer-2.17.2}/GameSentenceMiner/web/static/css/kanji-grid.css +0 -0
  79. {gamesentenceminer-2.17.1 → gamesentenceminer-2.17.2}/GameSentenceMiner/web/static/css/search.css +0 -0
  80. {gamesentenceminer-2.17.1 → gamesentenceminer-2.17.2}/GameSentenceMiner/web/static/favicon-96x96.png +0 -0
  81. {gamesentenceminer-2.17.1 → gamesentenceminer-2.17.2}/GameSentenceMiner/web/static/favicon.ico +0 -0
  82. {gamesentenceminer-2.17.1 → gamesentenceminer-2.17.2}/GameSentenceMiner/web/static/favicon.svg +0 -0
  83. {gamesentenceminer-2.17.1 → gamesentenceminer-2.17.2}/GameSentenceMiner/web/static/js/database.js +0 -0
  84. {gamesentenceminer-2.17.1 → gamesentenceminer-2.17.2}/GameSentenceMiner/web/static/js/kanji-grid.js +0 -0
  85. {gamesentenceminer-2.17.1 → gamesentenceminer-2.17.2}/GameSentenceMiner/web/static/js/search.js +0 -0
  86. {gamesentenceminer-2.17.1 → gamesentenceminer-2.17.2}/GameSentenceMiner/web/static/site.webmanifest +0 -0
  87. {gamesentenceminer-2.17.1 → gamesentenceminer-2.17.2}/GameSentenceMiner/web/static/style.css +0 -0
  88. {gamesentenceminer-2.17.1 → gamesentenceminer-2.17.2}/GameSentenceMiner/web/static/web-app-manifest-192x192.png +0 -0
  89. {gamesentenceminer-2.17.1 → gamesentenceminer-2.17.2}/GameSentenceMiner/web/static/web-app-manifest-512x512.png +0 -0
  90. {gamesentenceminer-2.17.1 → gamesentenceminer-2.17.2}/GameSentenceMiner/web/stats.py +0 -0
  91. {gamesentenceminer-2.17.1 → gamesentenceminer-2.17.2}/GameSentenceMiner/web/templates/components/navigation.html +0 -0
  92. {gamesentenceminer-2.17.1 → gamesentenceminer-2.17.2}/GameSentenceMiner/web/templates/components/theme-styles.html +0 -0
  93. {gamesentenceminer-2.17.1 → gamesentenceminer-2.17.2}/GameSentenceMiner/web/templates/database.html +0 -0
  94. {gamesentenceminer-2.17.1 → gamesentenceminer-2.17.2}/GameSentenceMiner/web/templates/search.html +0 -0
  95. {gamesentenceminer-2.17.1 → gamesentenceminer-2.17.2}/GameSentenceMiner/web/templates/utility.html +0 -0
  96. {gamesentenceminer-2.17.1 → gamesentenceminer-2.17.2}/GameSentenceMiner/wip/__init___.py +0 -0
  97. {gamesentenceminer-2.17.1 → gamesentenceminer-2.17.2}/GameSentenceMiner.egg-info/SOURCES.txt +0 -0
  98. {gamesentenceminer-2.17.1 → gamesentenceminer-2.17.2}/GameSentenceMiner.egg-info/dependency_links.txt +0 -0
  99. {gamesentenceminer-2.17.1 → gamesentenceminer-2.17.2}/GameSentenceMiner.egg-info/entry_points.txt +0 -0
  100. {gamesentenceminer-2.17.1 → gamesentenceminer-2.17.2}/GameSentenceMiner.egg-info/requires.txt +0 -0
  101. {gamesentenceminer-2.17.1 → gamesentenceminer-2.17.2}/GameSentenceMiner.egg-info/top_level.txt +0 -0
  102. {gamesentenceminer-2.17.1 → gamesentenceminer-2.17.2}/LICENSE +0 -0
  103. {gamesentenceminer-2.17.1 → gamesentenceminer-2.17.2}/README.md +0 -0
  104. {gamesentenceminer-2.17.1 → gamesentenceminer-2.17.2}/setup.cfg +0 -0
@@ -558,16 +558,40 @@ def start_monitoring_anki():
558
558
 
559
559
  # --- Anki Stats Kanji Extraction Utilities ---
560
560
 
561
- def get_all_anki_first_field_kanji():
561
+ def get_anki_earliest_date():
562
+ """
563
+ Fetches the earliest Anki card ID.
564
+ """
565
+ try:
566
+ note_ids = invoke("findCards", query="")
567
+ if not note_ids:
568
+ return 0
569
+
570
+ # Return the first card ID as the "earliest"
571
+ # return note_ids[0]
572
+ return min(note_ids)
573
+
574
+ except Exception as e:
575
+ logger.error(f"Failed to fetch kanji from Anki: {e}")
576
+ return 0
577
+
578
+ def get_all_anki_first_field_kanji(start_timestamp = None, end_timestamp = None):
562
579
  """
563
580
  Fetch all notes from Anki and extract unique kanji from the first field of each note.
564
581
  Returns a set of kanji characters.
582
+ Optional filtering by start_timestamp and end_timestamp on note IDs.
565
583
  """
566
584
  from GameSentenceMiner.web.stats import is_kanji
567
585
  try:
568
586
  note_ids = invoke("findNotes", query="")
569
587
  if not note_ids:
570
588
  return set()
589
+
590
+ # Filter note IDs by start and end timestamps if provided
591
+ if (start_timestamp and end_timestamp):
592
+ note_ids = [nid for nid in note_ids if int(start_timestamp) <= nid <= int(end_timestamp)]
593
+ if not note_ids:
594
+ return set()
571
595
  kanji_set = set()
572
596
  batch_size = 1000
573
597
  for i in range(0, len(note_ids), batch_size):
@@ -384,6 +384,7 @@ class ConfigApp:
384
384
  self.ocr_websocket_port_value = tk.StringVar(value=str(self.settings.advanced.ocr_websocket_port))
385
385
  self.texthooker_communication_websocket_port_value = tk.StringVar(value=str(self.settings.advanced.texthooker_communication_websocket_port))
386
386
  self.plaintext_websocket_export_port_value = tk.StringVar(value=str(self.settings.advanced.plaintext_websocket_port))
387
+ self.localhost_bind_address_value = tk.StringVar(value=self.settings.advanced.localhost_bind_address)
387
388
 
388
389
  # AI Settings
389
390
  self.ai_enabled_value = tk.BooleanVar(value=self.settings.ai.enabled)
@@ -407,6 +408,7 @@ class ConfigApp:
407
408
  self.overlay_engine_value = tk.StringVar(value=self.settings.overlay.engine)
408
409
  self.periodic_value = tk.BooleanVar(value=self.settings.overlay.periodic)
409
410
  self.periodic_interval_value = tk.StringVar(value=str(self.settings.overlay.periodic_interval))
411
+ self.scan_delay_value = tk.StringVar(value=str(self.settings.overlay.scan_delay))
410
412
 
411
413
  # Master Config Settings
412
414
  self.switch_to_default_if_not_found_value = tk.BooleanVar(value=self.master_config.switch_to_default_if_not_found)
@@ -621,6 +623,7 @@ class ConfigApp:
621
623
  ocr_websocket_port=int(self.ocr_websocket_port_value.get()),
622
624
  texthooker_communication_websocket_port=int(self.texthooker_communication_websocket_port_value.get()),
623
625
  plaintext_websocket_port=int(self.plaintext_websocket_export_port_value.get()),
626
+ localhost_bind_address=self.localhost_bind_address_value.get(),
624
627
  ),
625
628
  ai=Ai(
626
629
  enabled=self.ai_enabled_value.get(),
@@ -643,6 +646,7 @@ class ConfigApp:
643
646
  websocket_port=int(self.overlay_websocket_port_value.get()),
644
647
  monitor_to_capture=self.overlay_monitor.current() if self.monitors else 0,
645
648
  engine=OverlayEngine(self.overlay_engine_value.get()).value if self.overlay_engine_value.get() else OverlayEngine.LENS.value,
649
+ scan_delay=float(self.scan_delay_value.get()),
646
650
  periodic=self.periodic_value.get(),
647
651
  periodic_interval=self.periodic_interval_value.get(),
648
652
  )
@@ -1998,6 +2002,12 @@ class ConfigApp:
1998
2002
  ttk.Entry(advanced_frame, textvariable=self.polling_rate_value).grid(row=self.current_row, column=1, sticky='EW', pady=2)
1999
2003
  self.current_row += 1
2000
2004
 
2005
+ localhost_bind_address_i18n = advanced_i18n.get('localhost_bind_address', {})
2006
+ HoverInfoLabelWidget(advanced_frame, text=localhost_bind_address_i18n.get('label', 'LocalHost Bind Address:'),
2007
+ tooltip=localhost_bind_address_i18n.get('tooltip', 'Set this to 0.0.0.0 if you want to connect from another device in your LAN, otherwise leave as is.'), row=self.current_row, column=0)
2008
+ ttk.Entry(advanced_frame, textvariable=self.localhost_bind_address_value).grid(row=self.current_row, column=1, sticky='EW', pady=2)
2009
+ self.current_row += 1
2010
+
2001
2011
  current_ver_i18n = advanced_i18n.get('current_version', {})
2002
2012
  HoverInfoLabelWidget(advanced_frame, text=current_ver_i18n.get('label', 'Current Version:'), bootstyle="secondary",
2003
2013
  tooltip=current_ver_i18n.get('tooltip', '...'), row=self.current_row, column=0)
@@ -2285,7 +2295,15 @@ class ConfigApp:
2285
2295
  textvariable=self.overlay_engine_value)
2286
2296
  self.overlay_engine.grid(row=self.current_row, column=1, sticky='EW', pady=2)
2287
2297
  self.current_row += 1
2288
-
2298
+
2299
+ # Scan Delay
2300
+ scan_delay_i18n = overlay_i18n.get('scan_delay', {})
2301
+ HoverInfoLabelWidget(overlay_frame, text=scan_delay_i18n.get('label', 'Scan Delay:'),
2302
+ tooltip=scan_delay_i18n.get('tooltip', 'Delay between GSM Receiving Text, and Scanning for Overlay. Increase this value if your game\'s text appears slowly.'),
2303
+ row=self.current_row, column=0)
2304
+ ttk.Entry(overlay_frame, textvariable=self.scan_delay_value).grid(row=self.current_row, column=1, sticky='EW', pady=2)
2305
+ self.current_row += 1
2306
+
2289
2307
  # Periodic Settings
2290
2308
  periodic_i18n = overlay_i18n.get('periodic', {})
2291
2309
  HoverInfoLabelWidget(overlay_frame, text=periodic_i18n.get('label', 'Periodic:'),
@@ -1,21 +1,3 @@
1
- import asyncio
2
- import os
3
- import shutil
4
- import subprocess
5
- import sys
6
- import tempfile
7
- import threading
8
- import time
9
- import warnings
10
-
11
- import requests
12
-
13
- from GameSentenceMiner.util.get_overlay_coords import OverlayThread
14
- from GameSentenceMiner.util.gsm_utils import remove_html_and_cloze_tags
15
-
16
- os.environ.pop('TCL_LIBRARY', None)
17
-
18
-
19
1
  def handle_error_in_initialization(e):
20
2
  """Handle errors that occur during initialization."""
21
3
  logger.exception(e, exc_info=True)
@@ -30,9 +12,20 @@ def handle_error_in_initialization(e):
30
12
 
31
13
 
32
14
  try:
15
+ import asyncio
16
+ import os
17
+ import shutil
18
+ import subprocess
19
+ import sys
20
+ import tempfile
21
+ import threading
22
+ import time
23
+ import warnings
24
+ import requests
33
25
  import os.path
34
26
  import signal
35
27
  from subprocess import Popen
28
+ os.environ.pop('TCL_LIBRARY', None)
36
29
 
37
30
  import keyboard
38
31
  import ttkbootstrap as ttk
@@ -46,6 +39,8 @@ try:
46
39
  from GameSentenceMiner.util.configuration import logger, gsm_state, get_config, anki_results, AnkiUpdateResult, \
47
40
  get_temporary_directory, get_log_path, get_master_config, switch_profile_and_save, get_app_directory, gsm_status, \
48
41
  is_windows, is_linux
42
+ from GameSentenceMiner.util.get_overlay_coords import OverlayThread
43
+ from GameSentenceMiner.util.gsm_utils import remove_html_and_cloze_tags
49
44
 
50
45
  logger.debug(f"[Import] configuration: {time.time() - start_time:.3f}s")
51
46
 
@@ -701,5 +701,7 @@ def create_scene():
701
701
  if __name__ == '__main__':
702
702
  logging.basicConfig(level=logging.INFO)
703
703
  connect_to_obs_sync()
704
- # set_fit_to_screen_for_scene_items(get_current_scene())
705
- create_scene()
704
+ img = get_screenshot_PIL(source_name='Display Capture 2', compression=100, img_format='jpg', width=2560, height=1440)
705
+ img.show()
706
+ # # set_fit_to_screen_for_scene_items(get_current_scene())
707
+ # create_scene()
@@ -175,7 +175,7 @@ class WebsocketServerThread(threading.Thread):
175
175
  self._stop_event = stop_event = asyncio.Event()
176
176
  self._event.set()
177
177
  self.server = start_server = websockets.serve(self.server_handler,
178
- "0.0.0.0",
178
+ get_config().advanced.localhost_bind_address,
179
179
  get_config().advanced.ocr_websocket_port,
180
180
  max_size=1000000000)
181
181
  async with start_server:
@@ -57,7 +57,7 @@ import psutil
57
57
  from .ocr import * # noqa: F403
58
58
  from .config import Config
59
59
  from .screen_coordinate_picker import get_screen_selection
60
- from GameSentenceMiner.util.configuration import get_temporary_directory
60
+ from GameSentenceMiner.util.configuration import get_config, get_temporary_directory
61
61
 
62
62
  from skimage.metrics import structural_similarity as ssim
63
63
  from typing import Union
@@ -291,7 +291,7 @@ class WebsocketServerThread(threading.Thread):
291
291
  self._stop_event = stop_event = asyncio.Event()
292
292
  self._event.set()
293
293
  self.server = start_server = websockets.serve(
294
- self.server_handler, '0.0.0.0', config.get_general('websocket_port'), max_size=1000000000)
294
+ self.server_handler, get_config().advanced.localhost_bind_address, config.get_general('websocket_port'), max_size=1000000000)
295
295
  async with start_server:
296
296
  await stop_event.wait()
297
297
  asyncio.run(main())
@@ -589,9 +589,10 @@ class Advanced:
589
589
  multi_line_sentence_storage_field: str = ''
590
590
  ocr_websocket_port: int = 9002
591
591
  texthooker_communication_websocket_port: int = 55001
592
- afk_timer_seconds: int = 120
593
- session_gap_seconds: int = 3600
594
- streak_requirement_hours: float = 0.01 # 1 second required per day to keep your streak by default
592
+ afk_timer_seconds: int = 120 # LEGACY, not used anymore
593
+ session_gap_seconds: int = 3600 # LEGACY, not used anymore
594
+ streak_requirement_hours: float = 0.01 # LEGACY, not used anymore
595
+ localhost_bind_address: str = '127.0.0.1' # Default 127.0.0.1 for security, set to 0.0.0.0 to allow external connections
595
596
 
596
597
  def __post_init__(self):
597
598
  if self.plaintext_websocket_port == -1:
@@ -646,6 +647,7 @@ class Overlay:
646
647
  monitor_to_capture: int = 0
647
648
  periodic: bool = False
648
649
  periodic_interval: float = 1.0
650
+ scan_delay: float = 0.25
649
651
 
650
652
  def __post_init__(self):
651
653
  if self.monitor_to_capture == -1:
@@ -1248,5 +1250,5 @@ is_dev = is_running_from_source()
1248
1250
 
1249
1251
  is_beangate = os.path.exists("C:/Users/Beangate")
1250
1252
 
1251
- logger.debug(f"Running in development mode: {is_dev}")
1252
- logger.debug(f"Running on Beangate's PC: {is_beangate}")
1253
+ # logger.debug(f"Running in development mode: {is_dev}")
1254
+ # logger.debug(f"Running on Beangate's PC: {is_beangate}")
@@ -264,6 +264,50 @@ class SQLiteDBTable:
264
264
  @classmethod
265
265
  def drop(cls):
266
266
  cls._db.execute(f"DROP TABLE IF EXISTS {cls._table}", commit=True)
267
+
268
+ @classmethod
269
+ def has_column(cls, column_name: str) -> bool:
270
+ row = cls._db.fetchone(
271
+ f"PRAGMA table_info({cls._table})")
272
+ if not row:
273
+ return False
274
+ columns = [col[1] for col in cls._db.fetchall(
275
+ f"PRAGMA table_info({cls._table})")]
276
+ return column_name in columns
277
+
278
+ @classmethod
279
+ def rename_column(cls, old_column: str, new_column: str):
280
+ cls._db.execute(
281
+ f"ALTER TABLE {cls._table} RENAME COLUMN {old_column} TO {new_column}", commit=True)
282
+
283
+ @classmethod
284
+ def drop_column(cls, column_name: str):
285
+ cls._db.execute(
286
+ f"ALTER TABLE {cls._table} DROP COLUMN {column_name}", commit=True)
287
+
288
+ @classmethod
289
+ def get_column_type(cls, column_name: str) -> Optional[str]:
290
+ row = cls._db.fetchone(
291
+ f"PRAGMA table_info({cls._table})")
292
+ if not row:
293
+ return None
294
+ columns = cls._db.fetchall(
295
+ f"PRAGMA table_info({cls._table})")
296
+ for col in columns:
297
+ if col[1] == column_name:
298
+ return col[2] # Return the type
299
+ return None
300
+
301
+ @classmethod
302
+ def alter_column_type(cls, old_column: str, new_column: str, new_type: str):
303
+ # Add new column
304
+ cls._db.execute(
305
+ f"ALTER TABLE {cls._table} ADD COLUMN {new_column} {new_type}", commit=True)
306
+ # Copy and cast data
307
+ cls._db.execute(
308
+ f"UPDATE {cls._table} SET {new_column} = CAST({old_column} AS {new_type})", commit=True)
309
+ cls._db.execute(
310
+ f"ALTER TABLE {cls._table} DROP COLUMN {old_column}", commit=True)
267
311
 
268
312
 
269
313
  class AIModelsTable(SQLiteDBTable):
@@ -333,10 +377,10 @@ class AIModelsTable(SQLiteDBTable):
333
377
 
334
378
  class GameLinesTable(SQLiteDBTable):
335
379
  _table = 'game_lines'
336
- _fields = ['game_name', 'line_text', 'timestamp', 'screenshot_in_anki',
337
- 'audio_in_anki', 'screenshot_path', 'audio_path', 'replay_path', 'translation']
380
+ _fields = ['game_name', 'line_text', 'screenshot_in_anki',
381
+ 'audio_in_anki', 'screenshot_path', 'audio_path', 'replay_path', 'translation', 'timestamp']
338
382
  _types = [str, # Includes primary key type
339
- str, str, str, str, str, str, str, str, str]
383
+ str, str, str, str, str, str, str, str, float]
340
384
  _pk = 'id'
341
385
  _auto_increment = False # Use string IDs
342
386
 
@@ -355,7 +399,7 @@ class GameLinesTable(SQLiteDBTable):
355
399
  self.game_name = game_name
356
400
  self.line_text = line_text
357
401
  self.context = context
358
- self.timestamp = timestamp if timestamp is not None else datetime.now().timestamp()
402
+ self.timestamp = float(timestamp) if timestamp is not None else datetime.now().timestamp()
359
403
  self.screenshot_in_anki = screenshot_in_anki if screenshot_in_anki is not None else ''
360
404
  self.audio_in_anki = audio_in_anki if audio_in_anki is not None else ''
361
405
  self.screenshot_path = screenshot_path if screenshot_path is not None else ''
@@ -416,9 +460,79 @@ class GameLinesTable(SQLiteDBTable):
416
460
  params,
417
461
  commit=True
418
462
  )
463
+
464
+ @classmethod
465
+ def get_lines_filtered_by_timestamp(cls, start: Optional[float] = None, end: Optional[float] = None) -> List['GameLinesTable']:
466
+ """
467
+ Fetches all lines optionally filtered by start and end timestamps.
468
+ If start or end is None, that bound is ignored.
469
+ """
470
+ query = f"SELECT * FROM {cls._table}"
471
+ conditions = []
472
+ params = []
473
+
474
+ # Add timestamp conditions if provided
475
+ if start is not None:
476
+ conditions.append("timestamp >= ?")
477
+ params.append(start)
478
+ if end is not None:
479
+ conditions.append("timestamp <= ?")
480
+ params.append(end)
481
+
482
+ # Combine conditions into WHERE clause if any
483
+ if conditions:
484
+ query += " WHERE " + " AND ".join(conditions)
485
+
486
+ # Sort by timestamp ascending
487
+ query += " ORDER BY timestamp ASC"
488
+
489
+ # Execute the query
490
+ rows = cls._db.fetchall(query, tuple(params))
491
+ return [cls.from_row(row) for row in rows]
492
+
493
+ class StatsRollupTable(SQLiteDBTable):
494
+ _table = 'stats_rollup'
495
+ _fields = ['date', 'games_played', 'lines_mined', 'anki_cards_created', 'time_spent_mining']
496
+ _types = [int, # Includes primary key type
497
+ str, int, int, int, float]
498
+ _pk = 'id'
499
+ _auto_increment = True # Use auto-incrementing integer IDs
500
+
501
+ def __init__(self, id: Optional[int] = None,
502
+ date: Optional[str] = None,
503
+ games_played: int = 0,
504
+ lines_mined: int = 0,
505
+ anki_cards_created: int = 0,
506
+ time_spent_mining: float = 0.0):
507
+ self.id = id
508
+ self.date = date if date is not None else datetime.now().strftime("%Y-%m-%d")
509
+ self.games_played = games_played
510
+ self.lines_mined = lines_mined
511
+ self.anki_cards_created = anki_cards_created
512
+ self.time_spent_mining = time_spent_mining
419
513
 
514
+ @classmethod
515
+ def get_stats_for_date(cls, date: str) -> Optional['StatsRollupTable']:
516
+ row = cls._db.fetchone(
517
+ f"SELECT * FROM {cls._table} WHERE date=?", (date,))
518
+ return cls.from_row(row) if row else None
420
519
 
421
- def get_db_directory():
520
+ @classmethod
521
+ def update_stats(cls, date: str, games_played: int = 0, lines_mined: int = 0, anki_cards_created: int = 0, time_spent_mining: float = 0.0):
522
+ stats = cls.get_stats_for_date(date)
523
+ if not stats:
524
+ new_stats = cls(date=date, games_played=games_played,
525
+ lines_mined=lines_mined, anki_cards_created=anki_cards_created, time_spent_mining=time_spent_mining)
526
+ new_stats.save()
527
+ return
528
+ stats.games_played += games_played
529
+ stats.lines_mined += lines_mined
530
+ stats.anki_cards_created += anki_cards_created
531
+ stats.time_spent_mining += time_spent_mining
532
+ stats.save()
533
+
534
+ # Ensure database directory exists and return path
535
+ def get_db_directory(test=False, delete_test=False) -> str:
422
536
  if platform == 'win32': # Windows
423
537
  appdata_dir = os.getenv('APPDATA')
424
538
  else: # macOS and Linux
@@ -426,7 +540,11 @@ def get_db_directory():
426
540
  config_dir = os.path.join(appdata_dir, 'GameSentenceMiner')
427
541
  # Create the directory if it doesn't exist
428
542
  os.makedirs(config_dir, exist_ok=True)
429
- return os.path.join(config_dir, 'gsm.db')
543
+ path = os.path.join(config_dir, 'gsm.db' if not test else 'gsm_test.db')
544
+ if test and delete_test:
545
+ if os.path.exists(path):
546
+ os.remove(path)
547
+ return path
430
548
 
431
549
 
432
550
  # Backup and compress the database on load, with today's date, up to 5 days ago (clean up old backups)
@@ -462,6 +580,8 @@ db_path = get_db_directory()
462
580
  if os.path.exists(db_path):
463
581
  backup_db(db_path)
464
582
 
583
+ # db_path = get_db_directory(test=True, delete_test=False)
584
+
465
585
  gsm_db = SQLiteDB(db_path)
466
586
 
467
587
  for cls in [AIModelsTable, GameLinesTable]:
@@ -470,13 +590,47 @@ for cls in [AIModelsTable, GameLinesTable]:
470
590
  # cls.drop()
471
591
  # cls.set_db(gsm_db) # --- IGNORE ---
472
592
 
593
+ # GameLinesTable.drop_column('timestamp')
594
+
595
+ # if GameLinesTable.has_column('timestamp_old'):
596
+ # GameLinesTable.alter_column_type('timestamp_old', 'timestamp', 'TEXT')
597
+ # logger.info("Altered 'timestamp_old' column to 'timestamp' with TEXT type in GameLinesTable.")
598
+
599
+ def check_and_run_migrations():
600
+ def migrate_timestamp():
601
+ if GameLinesTable.has_column('timestamp') and GameLinesTable.get_column_type('timestamp') != 'REAL':
602
+ logger.info("Migrating 'timestamp' column to REAL type in GameLinesTable.")
603
+ # Rename 'timestamp' to 'timestamp_old'
604
+ GameLinesTable.rename_column('timestamp', 'timestamp_old')
605
+ # Copy and cast data from old column to new column
606
+ GameLinesTable.alter_column_type('timestamp_old', 'timestamp', 'REAL')
607
+ logger.info("Migrated 'timestamp' column to REAL type in GameLinesTable.")
608
+
609
+
610
+ migrate_timestamp()
611
+
612
+ check_and_run_migrations()
613
+
614
+ # all_lines = GameLinesTable.all()
615
+
616
+
617
+ # # Convert String timestamp to float timestamp
618
+ # for line in all_lines:
619
+ # if isinstance(line.timestamp, str):
620
+ # try:
621
+ # line.timestamp = float(line.timestamp)
622
+ # except ValueError:
623
+ # # Handle invalid timestamp format
624
+ # line.timestamp = 0.0
625
+ # line.save()
626
+
473
627
  # import random
474
628
  # import uuid
475
629
  # from datetime import datetime
476
630
  # from GameSentenceMiner.util.text_log import GameLine
477
631
  # from GameSentenceMiner.util.db import GameLinesTable
478
632
 
479
- # List of common Japanese characters (kanji, hiragana, katakana)
633
+ # # List of common Japanese characters (kanji, hiragana, katakana)
480
634
  # japanese_chars = (
481
635
  # "あいうえおかきくけこさしすせそたちつてとなにぬねのはひふへほまみむめもやゆよらりるれろわをん"
482
636
  # "アイウエオカキクケコサシスセソタチツテトナニヌネノハヒフヘホマミムメモヤユヨラリルレロワヲン"
@@ -513,6 +667,7 @@ for cls in [AIModelsTable, GameLinesTable]:
513
667
 
514
668
  # if len(lines_batch) >= batch_size:
515
669
  # GameLinesTable.add_lines(lines_batch)
670
+ # GameLinesTable2.add_lines(lines_batch)
516
671
  # lines_batch = []
517
672
  # if i % 1000 == 0:
518
673
  # print(f"Inserted {i} lines...")
@@ -520,5 +675,18 @@ for cls in [AIModelsTable, GameLinesTable]:
520
675
  # # Insert any remaining lines
521
676
  # if lines_batch:
522
677
  # GameLinesTable.add_lines(lines_batch)
678
+ # GameLinesTable2.add_lines(lines_batch)
679
+ # for _ in range(10): # Run multiple times to see consistent timing
680
+ # start_time = time.time()
681
+ # GameLinesTable.all()
682
+ # end_time = time.time()
683
+
684
+ # print(f"Time taken to query all lines from GameLinesTable: {end_time - start_time:.2f} seconds")
685
+
686
+ # start_time = time.time()
687
+ # GameLinesTable2.all()
688
+ # end_time = time.time()
689
+
690
+ # print(f"Time taken to query all lines from GameLinesTable2: {end_time - start_time:.2f} seconds")
523
691
 
524
- print("Done populating GameLinesDB with random Japanese text.")
692
+ # print("Done populating GameLinesTable and GameLinesTable2 with random Japanese text.")
@@ -71,6 +71,7 @@ def download_obs_if_needed():
71
71
  def get_windows_obs_url():
72
72
  machine = platform.machine().lower()
73
73
  if machine in ['arm64', 'aarch64']:
74
+ logger.info("Detected Windows on ARM64. Getting ARM64 version of OBS Studio.")
74
75
  return next(asset['browser_download_url'] for asset in latest_release['assets'] if
75
76
  asset['name'].endswith('Windows-arm64.zip'))
76
77
  return next(asset['browser_download_url'] for asset in latest_release['assets'] if
@@ -80,12 +81,12 @@ def download_obs_if_needed():
80
81
  with urllib.request.urlopen(latest_release_url) as response:
81
82
  latest_release = json.load(response)
82
83
  obs_url = {
83
- "Windows": get_windows_obs_url(),
84
- "Linux": next(asset['browser_download_url'] for asset in latest_release['assets'] if
85
- asset['name'].endswith('Ubuntu-24.04-x86_64.deb')),
86
- "Darwin": next(asset['browser_download_url'] for asset in latest_release['assets'] if
87
- asset['name'].endswith('macOS-Intel.dmg'))
88
- }.get(platform.system(), None)
84
+ "Windows": get_windows_obs_url,
85
+ # "Linux": lambda: next(asset['browser_download_url'] for asset in latest_release['assets'] if
86
+ # asset['name'].endswith('Ubuntu-24.04-x86_64.deb')),
87
+ # "Darwin": lambda: next(asset['browser_download_url'] for asset in latest_release['assets'] if
88
+ # asset['name'].endswith('macOS-Intel.dmg'))
89
+ }.get(platform.system(), lambda: None)()
89
90
 
90
91
  if obs_url is None:
91
92
  logger.error("Unsupported OS. Please install OBS manually.")
@@ -163,43 +164,75 @@ def download_ffmpeg_if_needed():
163
164
  logger.info("FFmpeg directory exists but executables are missing. Re-downloading FFmpeg...")
164
165
  shutil.rmtree(ffmpeg_dir)
165
166
 
166
- ffmpeg_url = {
167
- "Windows": "https://www.gyan.dev/ffmpeg/builds/ffmpeg-release-essentials.zip",
168
- "Linux": "https://johnvansickle.com/ffmpeg/releases/ffmpeg-release-amd64-static.tar.xz",
169
- "Darwin": "https://evermeet.cx/ffmpeg/ffmpeg.zip"
170
- }.get(platform.system(), None)
167
+ system = platform.system()
168
+ ffmpeg_url = None
169
+ compressed_format = "zip"
170
+ if system == "Windows":
171
+ machine = platform.machine().lower()
172
+ if machine in ['arm64', 'aarch64']:
173
+ ffmpeg_url = "https://gsm.beangate.us/ffmpeg-8.0-essentials-shared-win-arm64.zip"
174
+ compressed_format = "zip"
175
+ else:
176
+ ffmpeg_url = "https://www.gyan.dev/ffmpeg/builds/ffmpeg-release-essentials.zip"
177
+ compressed_format = "zip"
178
+ # elif system == "Linux":
179
+ # ffmpeg_url = "https://johnvansickle.com/ffmpeg/releases/ffmpeg-release-amd64-static.tar.xz"
180
+ # elif system == "Darwin":
181
+ # ffmpeg_url = "https://evermeet.cx/ffmpeg/ffmpeg.zip"
171
182
 
172
183
  if ffmpeg_url is None:
173
- logger.error("Unsupported OS. Please install FFmpeg manually.")
184
+ logger.error("Unsupported OS/architecture. Please install FFmpeg manually.")
174
185
  return
175
186
 
176
187
  download_dir = os.path.join(get_app_directory(), "downloads")
177
188
  os.makedirs(download_dir, exist_ok=True)
178
- ffmpeg_archive = os.path.join(download_dir, "ffmpeg.zip")
189
+ ffmpeg_archive = os.path.join(download_dir, f"ffmpeg.{compressed_format}")
179
190
 
180
191
  logger.info(f"Downloading FFmpeg from {ffmpeg_url}...")
181
192
  urllib.request.urlretrieve(ffmpeg_url, ffmpeg_archive)
182
193
  logger.info(f"FFmpeg downloaded. Extracting to {ffmpeg_dir}...")
183
194
 
184
195
  os.makedirs(ffmpeg_dir, exist_ok=True)
185
-
186
- with zipfile.ZipFile(ffmpeg_archive, 'r') as zip_ref:
187
- for member in zip_ref.namelist():
188
- filename = os.path.basename(member)
189
- if filename: # Skip directories
190
- source = zip_ref.open(member)
191
- target = open(os.path.join(ffmpeg_dir, filename), "wb")
192
- with source, target:
193
- shutil.copyfileobj(source, target)
196
+
197
+ # Extract 7z
198
+ # Extract archive
199
+ if ffmpeg_url.endswith('.7z'):
200
+ with py7zr.SevenZipFile(ffmpeg_archive, mode='r') as z:
201
+ z.extractall(ffmpeg_dir)
202
+ else:
203
+ with zipfile.ZipFile(ffmpeg_archive, 'r') as zip_ref:
204
+ zip_ref.extractall(ffmpeg_dir)
205
+
206
+ # Flatten directory structure - move all files to root ffmpeg_dir
207
+ def flatten_directory(directory):
208
+ for root, dirs, files in os.walk(directory):
209
+ for file in files:
210
+ file_path = os.path.join(root, file)
211
+ if root != directory: # Only move files from subdirectories
212
+ target_path = os.path.join(directory, file)
213
+ # Handle name conflicts by keeping the first occurrence
214
+ if not os.path.exists(target_path):
215
+ shutil.move(file_path, target_path)
216
+ # Remove empty subdirectories
217
+ for root, dirs, files in os.walk(directory, topdown=False):
218
+ for dir_name in dirs:
219
+ dir_path = os.path.join(root, dir_name)
220
+ try:
221
+ os.rmdir(dir_path)
222
+ except OSError:
223
+ pass # Directory not empty
224
+
225
+ flatten_directory(ffmpeg_dir)
194
226
 
195
227
  # Copy ffmpeg.exe to the python folder
196
228
  if os.path.exists(ffmpeg_exe_path):
197
229
  shutil.copy2(ffmpeg_exe_path, ffmpeg_in_python)
198
230
  logger.info(f"Copied ffmpeg.exe to Python folder: {ffmpeg_in_python}")
199
231
  else:
200
- logger.warning(f"ffmpeg.exe not found in {ffmpeg_dir}.")
232
+ logger.warning(f"ffmpeg.exe not found in {ffmpeg_dir}. Extraction might have failed.")
201
233
  logger.info(f"FFmpeg extracted to {ffmpeg_dir}.")
202
234
 
235
+
203
236
  def download_ocenaudio_if_needed():
204
237
  ocenaudio_dir = os.path.join(get_app_directory(), 'ocenaudio')
205
238
  ocenaudio_exe_path = os.path.join(ocenaudio_dir, 'ocenaudio.exe')
@@ -234,4 +267,4 @@ def main():
234
267
  download_ocenaudio_if_needed()
235
268
 
236
269
  if __name__ == "__main__":
237
- main()
270
+ main()
@@ -283,6 +283,9 @@ class OverlayProcessor:
283
283
  if not self.lens:
284
284
  logger.error("OCR engines are not initialized. Cannot perform OCR for Overlay.")
285
285
  return []
286
+
287
+ if get_config().overlay.scan_delay > 0:
288
+ await asyncio.sleep(get_config().overlay.scan_delay)
286
289
 
287
290
  # 1. Get screenshot
288
291
  full_screenshot, monitor_width, monitor_height = self._get_full_screenshot()