GameSentenceMiner 2.18.17__py3-none-any.whl → 2.18.19__py3-none-any.whl

GameSentenceMiner/util/db.py

@@ -15,10 +15,14 @@ import uuid
 import pytz
 from datetime import timedelta
 
+import regex
+
 from GameSentenceMiner.util.text_log import GameLine
 from GameSentenceMiner.util.configuration import get_stats_config, logger, is_dev
 import gzip
 
+# Matches any Unicode punctuation (\p{P}), symbol (\p{S}), or separator (\p{Z}); \p{Z} includes whitespace/separator chars
+punctuation_regex = regex.compile(r'[\p{P}\p{S}\p{Z}]')
 
 class SQLiteDB:
     """
@@ -136,7 +140,7 @@ class SQLiteDBTable:
         return cls.from_row(row) if row else None
 
     @classmethod
-    def from_row(cls: Type[T], row: Tuple) -> T:
+    def from_row(cls: Type[T], row: Tuple, clean_columns: list = []) -> T:
         if not row:
             return None
         obj = cls()
@@ -162,6 +166,9 @@ class SQLiteDBTable:
                 field = expected_fields[expected_pos]
                 field_type = cls._types[expected_pos]
                 
+                if field in clean_columns and isinstance(row_value, str):
+                    row_value = punctuation_regex.sub('', row_value).strip() 
+                
                 cls._set_field_value(obj, field, field_type, row_value, expected_pos == 0 and field == cls._pk)
                 
         except Exception as e:
@@ -483,6 +490,12 @@ class GameLinesTable(SQLiteDBTable):
         self.translation = translation if translation is not None else ''
         self.original_game_name = original_game_name if original_game_name is not None else ''
         self.game_id = game_id if game_id is not None else ''
+        
+    @classmethod
+    def all(cls, for_stats: bool = False) -> List['GameLinesTable']:
+        rows = cls._db.fetchall(f"SELECT * FROM {cls._table}")
+        clean_columns = ['line_text'] if for_stats else []
+        return [cls.from_row(row, clean_columns=clean_columns) for row in rows]
 
     @classmethod
     def get_all_lines_for_scene(cls, game_name: str) -> List['GameLinesTable']:
@@ -539,7 +552,7 @@ class GameLinesTable(SQLiteDBTable):
         )
         
     @classmethod
-    def get_lines_filtered_by_timestamp(cls, start: Optional[float] = None, end: Optional[float] = None) -> List['GameLinesTable']:
+    def get_lines_filtered_by_timestamp(cls, start: Optional[float] = None, end: Optional[float] = None, for_stats=False) -> List['GameLinesTable']:
         """
         Fetches all lines optionally filtered by start and end timestamps.
         If start or end is None, that bound is ignored.
@@ -565,8 +578,9 @@ class GameLinesTable(SQLiteDBTable):
 
         # Execute the query
         rows = cls._db.fetchall(query, tuple(params))
-        return [cls.from_row(row) for row in rows]
-        
+        clean_columns = ['line_text'] if for_stats else []
+        return [cls.from_row(row, clean_columns=clean_columns) for row in rows]
+
 class StatsRollupTable(SQLiteDBTable):
     _table = 'stats_rollup'
     _fields = ['date', 'games_played', 'lines_mined', 'anki_cards_created', 'time_spent_mining']

GameSentenceMiner/web/database_api.py

@@ -24,7 +24,6 @@ from GameSentenceMiner.web.stats import (
     calculate_hourly_reading_speed, calculate_peak_daily_stats, calculate_peak_session_stats
 )
 
-
 def register_database_api_routes(app):
     """Register all database API routes with the Flask app."""
     
@@ -1069,7 +1068,6 @@ def register_database_api_routes(app):
         Accepts optional 'year' parameter to filter heatmap data.
         """
         try:
-            punctionation_regex = regex.compile(r'[\p{P}\p{S}\p{Z}]')
             # Get optional year filter parameter
             filter_year = request.args.get('year', None)
 
@@ -1082,7 +1080,7 @@ def register_database_api_routes(app):
             end_timestamp = float(end_timestamp) if end_timestamp else None
 
             # 1. Fetch all lines and sort them chronologically
-            all_lines = GameLinesTable.get_lines_filtered_by_timestamp(start=start_timestamp, end=end_timestamp)
+            all_lines = GameLinesTable.get_lines_filtered_by_timestamp(start=start_timestamp, end=end_timestamp, for_stats=True)
             
             if not all_lines:
                 return jsonify({"labels": [], "datasets": []})
@@ -1095,14 +1093,12 @@ def register_database_api_routes(app):
                 day_str = datetime.date.fromtimestamp(float(line.timestamp)).strftime('%Y-%m-%d')
                 game = line.game_name or "Unknown Game"
                 # Remove punctuation and symbols from line text before counting characters
-                clean_text = punctionation_regex.sub('', str(line.line_text)) if line.line_text else ''
-                if not isinstance(clean_text, str) and not wrong_instance_found:
-                    logger.info(f"Non-string line_text encountered: {clean_text} (type: {type(clean_text)})")
+                if not isinstance(line.line_text, str) and not wrong_instance_found:
+                    logger.info(f"Non-string line_text encountered: {line.line_text} (type: {type(line.line_text)})")
                     wrong_instance_found = True
 
-                line.line_text = clean_text  # Update line text to cleaned version for future use
                 daily_data[day_str][game]['lines'] += 1
-                daily_data[day_str][game]['chars'] += len(clean_text)
+                daily_data[day_str][game]['chars'] += len(line.line_text)
 
             # 3. Create cumulative datasets for Chart.js
             sorted_days = sorted(daily_data.keys())
@@ -1318,7 +1314,7 @@ def register_database_api_routes(app):
             today = datetime.date.today()
             
             # Get all lines for overall progress
-            all_lines = GameLinesTable.all()
+            all_lines = GameLinesTable.all(for_stats=True)
             if not all_lines:
                 return jsonify({
                     'hours': {'required': 0, 'progress': 0, 'has_target': False},
@@ -1428,7 +1424,7 @@ def register_database_api_routes(app):
             thirty_days_ago = today - datetime.timedelta(days=30)
             
             # Get all lines
-            all_lines = GameLinesTable.all()
+            all_lines = GameLinesTable.all(for_stats=True)
             if not all_lines:
                 return jsonify({
                     'hours': {'projection': 0, 'daily_average': 0},

{gamesentenceminer-2.18.17.dist-info → gamesentenceminer-2.18.19.dist-info}/METADATA

@@ -1,7 +1,7 @@
 Metadata-Version: 2.4
 Name: GameSentenceMiner
-Version: 2.18.17
-Summary: A tool for mining sentences from games. Update: Overlay?
+Version: 2.18.19
+Summary: A tool for mining sentences from games. Fix Goals Char Count
 Author-email: Beangate <bpwhelan95@gmail.com>
 License: MIT License
 Project-URL: Homepage, https://github.com/bpwhelan/GameSentenceMiner
@@ -48,6 +48,7 @@ Requires-Dist: obsws-python~=1.7.2
 Requires-Dist: numpy==2.2.6
 Requires-Dist: regex
 Requires-Dist: faster-whisper~=1.2.0
+Requires-Dist: torchaudio==2.8.0
 Dynamic: license-file
 
 # GSM - An Immersion toolkit for Games.

{gamesentenceminer-2.18.17.dist-info → gamesentenceminer-2.18.19.dist-info}/RECORD

@@ -43,7 +43,7 @@ GameSentenceMiner/ui/screenshot_selector.py,sha256=7QvDhOMpA0ej8x_lYtu6fhmrWbM1G
 GameSentenceMiner/util/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 GameSentenceMiner/util/audio_player.py,sha256=-yFsf0qoTSS1ga5rCmEJZJGUSJzXCvfZHY3t0NxycDk,7896
 GameSentenceMiner/util/configuration.py,sha256=qndhFAN4oC1dawklllS3UBhK2DCVSTloGdZxDoTUGr4,48137
-GameSentenceMiner/util/db.py,sha256=CneZuFGIH6fosHblly4lcrWfU0Qjj5l0coxJy7m1igw,32237
+GameSentenceMiner/util/db.py,sha256=FQUvMHcQv_bRNE9LfrsFIxXXXog8BhOA1t4mr_UXoiI,33019
 GameSentenceMiner/util/electron_config.py,sha256=KfeJToeFFVw0IR5MKa-gBzpzaGrU-lyJbR9z-sDEHYU,8767
 GameSentenceMiner/util/ffmpeg.py,sha256=cAzztfY36Xf2WvsJDjavoiMOvA9ac2GVdCrSB4LzHk4,29007
 GameSentenceMiner/util/games_table.py,sha256=VM68MAsdyE6tpdwM4bDSk67qioBOvsEO8-TpnRmUnSo,12003
@@ -63,7 +63,7 @@ GameSentenceMiner/util/win10toast/__init__.py,sha256=6TL2w6rzNmpJEp6_v2cAJP_7ExA
 GameSentenceMiner/util/win10toast/__main__.py,sha256=5MYnBcFj8y_6Dyc1kiPd0_FsUuh4yl1cv5wsleU6V4w,668
 GameSentenceMiner/web/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 GameSentenceMiner/web/anki_api_endpoints.py,sha256=r30OTT3YVfgbF6aJ-EGWZLF-j2D9L63jLkRXMycU0p8,23681
-GameSentenceMiner/web/database_api.py,sha256=GyiMZWiT9q7fzA7D26YYE73DnAj3jCk9KUGT1P349y4,89996
+GameSentenceMiner/web/database_api.py,sha256=fLpVMZLn-LNwm-M0jNZLDSGH9dV6ml4zn28hNPHcOKY,89783
 GameSentenceMiner/web/events.py,sha256=6Vyz5c9MdpMIa7Zqljqhap2XFQnAVYJ0CdQV64TSZsA,5119
 GameSentenceMiner/web/gsm_websocket.py,sha256=B0VKpxmsRu0WRh5nFWlpDPBQ6-K2ed7TEIa0O6YWeoo,4166
 GameSentenceMiner/web/service.py,sha256=6cgUmDgtp3ZKzuPFszowjPoq-BDtC1bS3ux6sykeaqo,6662
@@ -135,9 +135,9 @@ GameSentenceMiner/web/templates/components/kanji_grid/thousand_character_classic
 GameSentenceMiner/web/templates/components/kanji_grid/wanikani_levels.json,sha256=8wjnnaYQqmho6t5tMxrIAc03512A2tYhQh5dfsQnfAM,11372
 GameSentenceMiner/web/templates/components/kanji_grid/words_hk_frequency_list.json,sha256=wRkqZNPzz6DT9OTPHpXwfqW96Qb96stCQNNgOL-ZdKk,17535
 GameSentenceMiner/wip/__init___.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-gamesentenceminer-2.18.17.dist-info/licenses/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
-gamesentenceminer-2.18.17.dist-info/METADATA,sha256=P3n2dpzmGimmdKfstygPWNvooSBB_lHKHgzNA961zA8,7488
-gamesentenceminer-2.18.17.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-gamesentenceminer-2.18.17.dist-info/entry_points.txt,sha256=2APEP25DbfjSxGeHtwBstMH8mulVhLkqF_b9bqzU6vQ,65
-gamesentenceminer-2.18.17.dist-info/top_level.txt,sha256=V1hUY6xVSyUEohb0uDoN4UIE6rUZ_JYx8yMyPGX4PgQ,18
-gamesentenceminer-2.18.17.dist-info/RECORD,,
+gamesentenceminer-2.18.19.dist-info/licenses/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
+gamesentenceminer-2.18.19.dist-info/METADATA,sha256=_cfxOe5CJVlJmfSSqWwFjFEVpkXs3gQiCgiQzNurcrc,7525
+gamesentenceminer-2.18.19.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+gamesentenceminer-2.18.19.dist-info/entry_points.txt,sha256=2APEP25DbfjSxGeHtwBstMH8mulVhLkqF_b9bqzU6vQ,65
+gamesentenceminer-2.18.19.dist-info/top_level.txt,sha256=V1hUY6xVSyUEohb0uDoN4UIE6rUZ_JYx8yMyPGX4PgQ,18
+gamesentenceminer-2.18.19.dist-info/RECORD,,