GameSentenceMiner 2.18.16__py3-none-any.whl → 2.18.18__py3-none-any.whl

GameSentenceMiner/owocr/owocr/ocr.py

@@ -111,8 +111,9 @@ def post_process(text, keep_blank_lines=False):
         text = '\n'.join([''.join(i.split()) for i in text.splitlines()])
     else:
         text = ''.join([''.join(i.split()) for i in text.splitlines()])
-    text = text.replace('…', '...')
-    text = re.sub('[・.]{2,}', lambda x: (x.end() - x.start()) * '.', text)
+    text = text.replace('…', '・・・')
+    text = re.sub('[・.]{2,}', lambda x: (x.end() - x.start()) * '・', text)
+    text = re.sub(r'・{3,}', '・・・', text)
     text = jaconv.h2z(text, ascii=True, digit=True)
     return text
 

GameSentenceMiner/owocr/owocr/run.py

@@ -1392,7 +1392,11 @@ def process_and_write_results(img_or_path, write_to=None, last_result=None, filt
     # print(engine_index)
 
     if res:
-        text = do_configured_ocr_replacements(text)
+        if isinstance(text, list):
+            for i, line in enumerate(text):
+                text[i] = do_configured_ocr_replacements(line)
+        else:
+            text = do_configured_ocr_replacements(text)
         if filtering:
             text, orig_text = filtering(text, last_result, engine=engine, is_second_ocr=is_second_ocr)
         if get_ocr_language() == "ja" or get_ocr_language() == "zh":

GameSentenceMiner/util/configuration.py

@@ -888,15 +888,16 @@ class Config:
                 if profile.advanced.streak_requirement_hours != default_stats.streak_requirement_hours:
                     self.stats.streak_requirement_hours = profile.advanced.streak_requirement_hours
         
+        self.overlay = self.get_config().overlay
+        
         # Add a way to migrate certain things based on version if needed, also help with better defaults
         if self.version:
-            if self.version != get_current_version():
+            current_version = get_current_version()
+            if self.version != current_version:
                 from packaging import version
-                logger.info(f"New Config Found: {self.version} != {get_current_version()}")
+                logger.info(f"New Config Found: {self.version} != {current_version}")
                 # Handle version mismatch
-                changed = False
                 if version.parse(self.version) < version.parse("2.18.0"):
-                    changed = True
                     # Example, doesn't need to be done
                     for profile in self.configs.values():
                         profile.obs.get_game_from_scene = True
@@ -904,11 +905,7 @@ class Config:
                         if profile.vad.selected_vad_model == WHISPER and profile.vad.backup_vad_model == SILERO:
                             profile.vad.backup_vad_model = OFF
 
-                if changed:
-                    self.save()
-        self.overlay = self.get_config().overlay
-
-        self.version = get_current_version()
+                self.save()
 
     def save(self):
         with open(get_config_path(), 'w') as file:

GameSentenceMiner/util/db.py

@@ -15,10 +15,14 @@ import uuid
 import pytz
 from datetime import timedelta
 
+import regex
+
 from GameSentenceMiner.util.text_log import GameLine
 from GameSentenceMiner.util.configuration import get_stats_config, logger, is_dev
 import gzip
 
+# Matches any Unicode punctuation (\p{P}), symbol (\p{S}), or separator (\p{Z}); \p{Z} includes whitespace/separator chars
+punctuation_regex = regex.compile(r'[\p{P}\p{S}\p{Z}]')
 
 class SQLiteDB:
     """
@@ -136,7 +140,7 @@ class SQLiteDBTable:
         return cls.from_row(row) if row else None
 
     @classmethod
-    def from_row(cls: Type[T], row: Tuple) -> T:
+    def from_row(cls: Type[T], row: Tuple, clean_columns: list = []) -> T:
         if not row:
             return None
         obj = cls()
@@ -162,6 +166,9 @@ class SQLiteDBTable:
                 field = expected_fields[expected_pos]
                 field_type = cls._types[expected_pos]
                 
+                if field in clean_columns and isinstance(row_value, str):
+                    row_value = punctuation_regex.sub('', row_value).strip() 
+                
                 cls._set_field_value(obj, field, field_type, row_value, expected_pos == 0 and field == cls._pk)
                 
         except Exception as e:
@@ -483,6 +490,12 @@ class GameLinesTable(SQLiteDBTable):
         self.translation = translation if translation is not None else ''
         self.original_game_name = original_game_name if original_game_name is not None else ''
         self.game_id = game_id if game_id is not None else ''
+        
+    @classmethod
+    def all(cls, for_stats: bool = False) -> List['GameLinesTable']:
+        rows = cls._db.fetchall(f"SELECT * FROM {cls._table}")
+        clean_columns = ['line_text'] if for_stats else []
+        return [cls.from_row(row, clean_columns=clean_columns) for row in rows]
 
     @classmethod
     def get_all_lines_for_scene(cls, game_name: str) -> List['GameLinesTable']:
@@ -539,7 +552,7 @@ class GameLinesTable(SQLiteDBTable):
         )
         
     @classmethod
-    def get_lines_filtered_by_timestamp(cls, start: Optional[float] = None, end: Optional[float] = None) -> List['GameLinesTable']:
+    def get_lines_filtered_by_timestamp(cls, start: Optional[float] = None, end: Optional[float] = None, for_stats=False) -> List['GameLinesTable']:
         """
         Fetches all lines optionally filtered by start and end timestamps.
         If start or end is None, that bound is ignored.
@@ -565,8 +578,9 @@ class GameLinesTable(SQLiteDBTable):
 
         # Execute the query
         rows = cls._db.fetchall(query, tuple(params))
-        return [cls.from_row(row) for row in rows]
-        
+        clean_columns = ['line_text'] if for_stats else []
+        return [cls.from_row(row, clean_columns=clean_columns) for row in rows]
+
 class StatsRollupTable(SQLiteDBTable):
     _table = 'stats_rollup'
     _fields = ['date', 'games_played', 'lines_mined', 'anki_cards_created', 'time_spent_mining']

GameSentenceMiner/web/database_api.py

@@ -24,7 +24,6 @@ from GameSentenceMiner.web.stats import (
     calculate_hourly_reading_speed, calculate_peak_daily_stats, calculate_peak_session_stats
 )
 
-
 def register_database_api_routes(app):
     """Register all database API routes with the Flask app."""
     
@@ -1069,7 +1068,6 @@ def register_database_api_routes(app):
         Accepts optional 'year' parameter to filter heatmap data.
         """
         try:
-            punctionation_regex = regex.compile(r'[\p{P}\p{S}\p{Z}]')
             # Get optional year filter parameter
             filter_year = request.args.get('year', None)
 
@@ -1082,7 +1080,7 @@ def register_database_api_routes(app):
             end_timestamp = float(end_timestamp) if end_timestamp else None
 
             # 1. Fetch all lines and sort them chronologically
-            all_lines = GameLinesTable.get_lines_filtered_by_timestamp(start=start_timestamp, end=end_timestamp)
+            all_lines = GameLinesTable.get_lines_filtered_by_timestamp(start=start_timestamp, end=end_timestamp, for_stats=True)
             
             if not all_lines:
                 return jsonify({"labels": [], "datasets": []})
@@ -1095,14 +1093,12 @@ def register_database_api_routes(app):
                 day_str = datetime.date.fromtimestamp(float(line.timestamp)).strftime('%Y-%m-%d')
                 game = line.game_name or "Unknown Game"
                 # Remove punctuation and symbols from line text before counting characters
-                clean_text = punctionation_regex.sub('', str(line.line_text)) if line.line_text else ''
-                if not isinstance(clean_text, str) and not wrong_instance_found:
-                    logger.info(f"Non-string line_text encountered: {clean_text} (type: {type(clean_text)})")
+                if not isinstance(line.line_text, str) and not wrong_instance_found:
+                    logger.info(f"Non-string line_text encountered: {line.line_text} (type: {type(line.line_text)})")
                     wrong_instance_found = True
 
-                line.line_text = clean_text  # Update line text to cleaned version for future use
                 daily_data[day_str][game]['lines'] += 1
-                daily_data[day_str][game]['chars'] += len(clean_text)
+                daily_data[day_str][game]['chars'] += len(line.line_text)
 
             # 3. Create cumulative datasets for Chart.js
             sorted_days = sorted(daily_data.keys())
@@ -1318,7 +1314,7 @@ def register_database_api_routes(app):
             today = datetime.date.today()
             
             # Get all lines for overall progress
-            all_lines = GameLinesTable.all()
+            all_lines = GameLinesTable.all(for_stats=True)
             if not all_lines:
                 return jsonify({
                     'hours': {'required': 0, 'progress': 0, 'has_target': False},
@@ -1428,7 +1424,7 @@ def register_database_api_routes(app):
             thirty_days_ago = today - datetime.timedelta(days=30)
             
             # Get all lines
-            all_lines = GameLinesTable.all()
+            all_lines = GameLinesTable.all(for_stats=True)
             if not all_lines:
                 return jsonify({
                     'hours': {'projection': 0, 'daily_average': 0},

{gamesentenceminer-2.18.16.dist-info → gamesentenceminer-2.18.18.dist-info}/METADATA

@@ -1,7 +1,7 @@
 Metadata-Version: 2.4
 Name: GameSentenceMiner
-Version: 2.18.16
-Summary: A tool for mining sentences from games. Update: Overlay?
+Version: 2.18.18
+Summary: A tool for mining sentences from games. Fix Goals Char Count
 Author-email: Beangate <bpwhelan95@gmail.com>
 License: MIT License
 Project-URL: Homepage, https://github.com/bpwhelan/GameSentenceMiner

{gamesentenceminer-2.18.16.dist-info → gamesentenceminer-2.18.18.dist-info}/RECORD

@@ -27,8 +27,8 @@ GameSentenceMiner/owocr/owocr/__init__.py,sha256=87hfN5u_PbL_onLfMACbc0F5j4KyIK9
 GameSentenceMiner/owocr/owocr/__main__.py,sha256=XQaqZY99EKoCpU-gWQjNbTs7Kg17HvBVE7JY8LqIE0o,157
 GameSentenceMiner/owocr/owocr/config.py,sha256=qM7kISHdUhuygGXOxmgU6Ef2nwBShrZtdqu4InDCViE,8103
 GameSentenceMiner/owocr/owocr/lens_betterproto.py,sha256=oNoISsPilVVRBBPVDtb4-roJtAhp8ZAuFTci3TGXtMc,39141
-GameSentenceMiner/owocr/owocr/ocr.py,sha256=8cqZEUF90UlV3jBIvxKBga6YBFGjNBCVu1UiBcwISG0,72215
-GameSentenceMiner/owocr/owocr/run.py,sha256=Z7VkoFrsoQbMTHc6CmwpcMzsOROK9A_RJRwhlxw15oA,81871
+GameSentenceMiner/owocr/owocr/ocr.py,sha256=XR6tbcj8ctDXn8NlpXrRZIel60zj2h3R0NKWBtEE5M4,72273
+GameSentenceMiner/owocr/owocr/run.py,sha256=z3EaF_a5m9T_ZrELYoaAzHPqzTO0cd7MQCndcnWXq_4,82035
 GameSentenceMiner/owocr/owocr/screen_coordinate_picker.py,sha256=Na6XStbQBtpQUSdbN3QhEswtKuU1JjReFk_K8t5ezQE,3395
 GameSentenceMiner/tools/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 GameSentenceMiner/tools/audio_offset_selector.py,sha256=8Stk3BP-XVIuzRv9nl9Eqd2D-1yD3JrgU-CamBywJmY,8542
@@ -42,8 +42,8 @@ GameSentenceMiner/ui/furigana_filter_preview.py,sha256=DAT2-j6vSDHr9ufk6PiaLikEs
 GameSentenceMiner/ui/screenshot_selector.py,sha256=7QvDhOMpA0ej8x_lYtu6fhmrWbM1GCg-dps3XVWwk1Q,8234
 GameSentenceMiner/util/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 GameSentenceMiner/util/audio_player.py,sha256=-yFsf0qoTSS1ga5rCmEJZJGUSJzXCvfZHY3t0NxycDk,7896
-GameSentenceMiner/util/configuration.py,sha256=8cXpeUJ4hOr8Qd4JPAx1qn_phBIFuKR5D-PybThu-Qk,48233
-GameSentenceMiner/util/db.py,sha256=CneZuFGIH6fosHblly4lcrWfU0Qjj5l0coxJy7m1igw,32237
+GameSentenceMiner/util/configuration.py,sha256=qndhFAN4oC1dawklllS3UBhK2DCVSTloGdZxDoTUGr4,48137
+GameSentenceMiner/util/db.py,sha256=FQUvMHcQv_bRNE9LfrsFIxXXXog8BhOA1t4mr_UXoiI,33019
 GameSentenceMiner/util/electron_config.py,sha256=KfeJToeFFVw0IR5MKa-gBzpzaGrU-lyJbR9z-sDEHYU,8767
 GameSentenceMiner/util/ffmpeg.py,sha256=cAzztfY36Xf2WvsJDjavoiMOvA9ac2GVdCrSB4LzHk4,29007
 GameSentenceMiner/util/games_table.py,sha256=VM68MAsdyE6tpdwM4bDSk67qioBOvsEO8-TpnRmUnSo,12003
@@ -63,7 +63,7 @@ GameSentenceMiner/util/win10toast/__init__.py,sha256=6TL2w6rzNmpJEp6_v2cAJP_7ExA
 GameSentenceMiner/util/win10toast/__main__.py,sha256=5MYnBcFj8y_6Dyc1kiPd0_FsUuh4yl1cv5wsleU6V4w,668
 GameSentenceMiner/web/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 GameSentenceMiner/web/anki_api_endpoints.py,sha256=r30OTT3YVfgbF6aJ-EGWZLF-j2D9L63jLkRXMycU0p8,23681
-GameSentenceMiner/web/database_api.py,sha256=GyiMZWiT9q7fzA7D26YYE73DnAj3jCk9KUGT1P349y4,89996
+GameSentenceMiner/web/database_api.py,sha256=fLpVMZLn-LNwm-M0jNZLDSGH9dV6ml4zn28hNPHcOKY,89783
 GameSentenceMiner/web/events.py,sha256=6Vyz5c9MdpMIa7Zqljqhap2XFQnAVYJ0CdQV64TSZsA,5119
 GameSentenceMiner/web/gsm_websocket.py,sha256=B0VKpxmsRu0WRh5nFWlpDPBQ6-K2ed7TEIa0O6YWeoo,4166
 GameSentenceMiner/web/service.py,sha256=6cgUmDgtp3ZKzuPFszowjPoq-BDtC1bS3ux6sykeaqo,6662
@@ -135,9 +135,9 @@ GameSentenceMiner/web/templates/components/kanji_grid/thousand_character_classic
 GameSentenceMiner/web/templates/components/kanji_grid/wanikani_levels.json,sha256=8wjnnaYQqmho6t5tMxrIAc03512A2tYhQh5dfsQnfAM,11372
 GameSentenceMiner/web/templates/components/kanji_grid/words_hk_frequency_list.json,sha256=wRkqZNPzz6DT9OTPHpXwfqW96Qb96stCQNNgOL-ZdKk,17535
 GameSentenceMiner/wip/__init___.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-gamesentenceminer-2.18.16.dist-info/licenses/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
-gamesentenceminer-2.18.16.dist-info/METADATA,sha256=X02yygMguMepPkrv_08yYZLA9v_SzhAwbwAfKFHhJ4U,7488
-gamesentenceminer-2.18.16.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-gamesentenceminer-2.18.16.dist-info/entry_points.txt,sha256=2APEP25DbfjSxGeHtwBstMH8mulVhLkqF_b9bqzU6vQ,65
-gamesentenceminer-2.18.16.dist-info/top_level.txt,sha256=V1hUY6xVSyUEohb0uDoN4UIE6rUZ_JYx8yMyPGX4PgQ,18
-gamesentenceminer-2.18.16.dist-info/RECORD,,
+gamesentenceminer-2.18.18.dist-info/licenses/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
+gamesentenceminer-2.18.18.dist-info/METADATA,sha256=Iyd8Jbyw4qbRxOd8MQD2ABDh0JP2v_qwqEeK69nrefE,7492
+gamesentenceminer-2.18.18.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+gamesentenceminer-2.18.18.dist-info/entry_points.txt,sha256=2APEP25DbfjSxGeHtwBstMH8mulVhLkqF_b9bqzU6vQ,65
+gamesentenceminer-2.18.18.dist-info/top_level.txt,sha256=V1hUY6xVSyUEohb0uDoN4UIE6rUZ_JYx8yMyPGX4PgQ,18
+gamesentenceminer-2.18.18.dist-info/RECORD,,