GameSentenceMiner 2.16.4__py3-none-any.whl → 2.16.6__py3-none-any.whl

GameSentenceMiner/anki.py

@@ -87,13 +87,18 @@ def update_anki_card(last_note: AnkiCard, note=None, audio_path='', video_path='
 
     if update_picture and screenshot_in_anki:
         note['fields'][get_config().anki.picture_field] = image_html
-        
+
     if video_in_anki:
         note['fields'][get_config().anki.video_field] = video_in_anki
-        
+
     if not get_config().screenshot.enabled:
         logger.info("Skipping Adding Screenshot to Anki, Screenshot is disabled in settings")
 
+    # Add game name to field if configured
+    game_name_field = get_config().anki.game_name_field
+    if note and 'fields' in note and game_name_field:
+        note['fields'][game_name_field] = get_current_game()
+
     if note and 'fields' in note and get_config().ai.enabled:
         sentence_field = note['fields'].get(get_config().anki.sentence_field, {})
         sentence_to_translate = sentence_field if sentence_field else last_note.get_field(

GameSentenceMiner/config_gui.py

@@ -307,6 +307,7 @@ class ConfigApp:
         self.word_field_value = tk.StringVar(value=self.settings.anki.word_field)
         self.previous_sentence_field_value = tk.StringVar(value=self.settings.anki.previous_sentence_field)
         self.previous_image_field_value = tk.StringVar(value=self.settings.anki.previous_image_field)
+        self.game_name_field_value = tk.StringVar(value=self.settings.anki.game_name_field)
         self.video_field_value = tk.StringVar(value=self.settings.anki.video_field)
         self.custom_tags_value = tk.StringVar(value=', '.join(self.settings.anki.custom_tags))
         self.tags_to_check_value = tk.StringVar(value=', '.join(self.settings.anki.tags_to_check))
@@ -440,6 +441,7 @@ class ConfigApp:
         default_category_config = getattr(self.default_settings, category)
 
         setattr(self.settings, category, default_category_config)
+        self.create_vars()  # Recreate variables to reflect default values
         recreate_tab()
         self.save_settings(profile_change=False)
         self.reload_settings()
@@ -527,6 +529,7 @@ class ConfigApp:
                 previous_sentence_field=self.previous_sentence_field_value.get(),
                 previous_image_field=self.previous_image_field_value.get(),
                 video_field=self.video_field_value.get(),
+                game_name_field=self.game_name_field_value.get(),
                 custom_tags=[tag.strip() for tag in self.custom_tags_value.get().split(',') if tag.strip()],
                 tags_to_check=[tag.strip().lower() for tag in self.tags_to_check_value.get().split(',') if tag.strip()],
                 add_game_tag=self.add_game_tag_value.get(),
@@ -1146,6 +1149,9 @@ class ConfigApp:
         HoverInfoLabelWidget(vad_frame, text=use_cpu_i18n.get('label', 'Force CPU'), tooltip=use_cpu_i18n.get('tooltip', 'Even if CUDA is installed, use CPU for Whisper'), row=self.current_row, column=0)
         ttk.Checkbutton(vad_frame, variable=self.use_cpu_for_inference_value, bootstyle="round-toggle").grid(row=self.current_row, column=1, sticky='W', pady=2)
         self.current_row += 1
+        
+        # Add Reset Button
+        self.add_reset_button(vad_frame, "vad", self.current_row, column=0, recreate_tab=self.create_vad_tab)
 
     @new_tab
     def create_paths_tab(self):
@@ -1319,6 +1325,12 @@ class ConfigApp:
                              row=self.current_row, column=0)
         ttk.Entry(anki_frame, textvariable=self.video_field_value).grid(row=self.current_row, column=1, sticky='EW', pady=2)
         self.current_row += 1
+        
+        game_name_field_i18n = anki_i18n.get('game_name_field', {})
+        HoverInfoLabelWidget(anki_frame, text=game_name_field_i18n.get('label', 'Game Name Field:'),
+                    tooltip=game_name_field_i18n.get('tooltip', 'Field in Anki for the game name.'), row=self.current_row, column=0)
+        ttk.Entry(anki_frame, textvariable=self.game_name_field_value).grid(row=self.current_row, column=1, columnspan=3, sticky='EW', pady=2)
+        self.current_row += 1
 
         tags_i18n = anki_i18n.get('custom_tags', {})
         HoverInfoLabelWidget(anki_frame, text=tags_i18n.get('label', '...'), tooltip=tags_i18n.get('tooltip', '...'),

GameSentenceMiner/locales/en_us.json

@@ -184,12 +184,16 @@
           },
           "video_field": {
             "label": "Video Field:",
-            "tooltip": "Field in Anki for associated videos. This will be AV1 encoded video of the VAD Trimmed Voiceline, if no Voice found, this will be empty."
+            "tooltip": "Field in Anki for associated videos. This will be AV1 encoded video of the VAD Trimmed Voiceline, if no Voice found, this will be empty. (OPTIONAL)"
           },
           "custom_tags": {
             "label": "Add Tags:",
             "tooltip": "Comma-separated custom tags for the Anki cards."
           },
+          "game_name_field": {
+            "label": "Game Name Field:",
+            "tooltip": "Field in Anki for the game name. If empty, game name will not be added as a field. (OPTIONAL)"
+          },
           "tags_to_check": {
             "label": "Tags to work on:",
             "tooltip": "Comma-separated Tags, script will only do 1-click on cards with these tags (Recommend keep empty, or use Yomitan Profile to add custom tag from texthooker page)"

GameSentenceMiner/locales/ja_jp.json

@@ -189,6 +189,10 @@
             "label": "追加タグ:",
             "tooltip": "Ankiカードに追加するカスタムタグ（カンマ区切り）。"
           },
+          "game_name_field": {
+            "label": "ゲーム名フィールド:",
+            "tooltip": "Ankiのゲーム名用フィールド。空欄の場合は追加されません。"
+          },
           "tags_to_check": {
             "label": "対象タグ:",
             "tooltip": "これらのタグを持つカードのみワンクリック対象になります（通常は空を推奨）。"

GameSentenceMiner/locales/zh_cn.json

@@ -190,6 +190,10 @@
             "label": "添加标签:",
             "tooltip": "Anki 卡片的自定义标签（以逗号分隔）。"
           },
+          "game_name_field": {
+            "label": "游戏名称字段:",
+            "tooltip": "Anki 中用于游戏名称的字段。如果为空，则不会添加游戏名称。"
+          },
           "tags_to_check": {
             "label": "处理的标签:",
             "tooltip": "脚本将只对带有这些标签的卡片进行一键操作（建议留空）。"

GameSentenceMiner/util/configuration.py

@@ -441,6 +441,7 @@ class Anki:
     custom_tags: List[str] = None
     tags_to_check: List[str] = None
     add_game_tag: bool = True
+    game_name_field: str = ''
     polling_rate: int = 200
     overwrite_audio: bool = False
     overwrite_picture: bool = True
@@ -780,6 +781,15 @@ class ProfileConfig:
     def config_changed(self, new: 'ProfileConfig') -> bool:
         return self != new
 
+@dataclass_json
+@dataclass
+class StatsConfig:
+    afk_timer_seconds: int = 120
+    session_gap_seconds: int = 3600
+    streak_requirement_hours: float = 0.01 # 1 second required per day to keep your streak by default
+    reading_hours_target: int = 1500  # Target reading hours based on TMW N1 achievement data
+    character_count_target: int = 25000000  # Target character count (25M) inspired by Discord server milestones
+    games_target: int = 100  # Target VNs/games completed based on Refold community standards
 
 @dataclass_json
 @dataclass
@@ -788,6 +798,7 @@ class Config:
     current_profile: str = DEFAULT_CONFIG
     switch_to_default_if_not_found: bool = True
     locale: str = Locale.English.value
+    stats: StatsConfig = field(default_factory=StatsConfig)
 
     @classmethod
     def new(cls):
@@ -812,6 +823,18 @@ class Config:
                 return cls.from_dict(data)
         else:
             return cls.new()
+        
+    def __post_init__(self):
+        # Move Stats to global config if found in profiles for legacy support
+        default_stats = StatsConfig()
+        for profile in self.configs.values():
+            if profile.advanced:
+                if profile.advanced.afk_timer_seconds != default_stats.afk_timer_seconds:
+                    self.stats.afk_timer_seconds = profile.advanced.afk_timer_seconds
+                if profile.advanced.session_gap_seconds != default_stats.session_gap_seconds:
+                    self.stats.session_gap_seconds = profile.advanced.session_gap_seconds
+                if profile.advanced.streak_requirement_hours != default_stats.streak_requirement_hours:
+                    self.stats.streak_requirement_hours = profile.advanced.streak_requirement_hours
 
     def save(self):
         with open(get_config_path(), 'w') as file:
@@ -1069,6 +1092,12 @@ def reload_config():
         logger.warning(
             "Backfill is enabled, but full auto is also enabled. Disabling backfill...")
         config.features.backfill_audio = False
+        
+def get_stats_config():
+    global config_instance
+    if config_instance is None:
+        config_instance = load_config()
+    return config_instance.stats
 
 
 def get_master_config():
@@ -1085,6 +1114,12 @@ def save_current_config(config):
     config_instance.set_config_for_profile(
         config_instance.current_profile, config)
     save_full_config(config_instance)
+    
+
+def save_stats_config(stats_config):
+    global config_instance
+    config_instance.stats = stats_config
+    save_full_config(config_instance)
 
 
 def switch_profile_and_save(profile_name):

GameSentenceMiner/util/ffmpeg.py

@@ -453,7 +453,7 @@ def trim_audio_based_on_last_line(untrimmed_audio, video_path, game_line, next_l
     ffmpeg_command = ffmpeg_base_command_list + [
         "-i", untrimmed_audio,
         "-ss", str(start_trim_time)]
-    if next_line and next_line > game_line.time:
+    if next_line and next_line > game_line.time and total_seconds:
         end_trim_seconds = total_seconds + (next_line - game_line.time).total_seconds() + get_config().audio.pre_vad_end_offset
         end_trim_time = f"{end_trim_seconds:.3f}"
         ffmpeg_command.extend(['-to', end_trim_time])
@@ -495,9 +495,9 @@ def get_video_timings(video_path, game_line, anki_card_creation_time=None):
     total_seconds = file_length - time_delta.total_seconds()
     total_seconds_after_offset = total_seconds + get_config().audio.beginning_offset
     if total_seconds < 0 or total_seconds >= file_length:
-        logger.error("Line mined is outside of the replay buffer! Defaulting to the beginning of the replay buffer. ")
+        logger.error("Line mined is outside of the replay buffer! Defaulting to the last 30 seconds of the replay buffer.")
         logger.info("Recommend either increasing replay buffer length in OBS Settings or mining faster.")
-        return 0, 0, 0, file_length
+        return max(file_length - 30, 0), 0, max(file_length - 30, 0), file_length
 
     return total_seconds_after_offset, total_seconds, total_seconds_after_offset, file_length
 

GameSentenceMiner/web/database_api.py

@@ -1,14 +1,17 @@
+import copy
 import datetime
 import re
 import csv
 import io
 from collections import defaultdict
+import time
 
 import flask
 from flask import request, jsonify
+import regex
 
 from GameSentenceMiner.util.db import GameLinesTable
-from GameSentenceMiner.util.configuration import logger, get_config, save_current_config
+from GameSentenceMiner.util.configuration import get_stats_config, logger, get_config, save_current_config, save_stats_config
 from GameSentenceMiner.web.stats import (
     calculate_kanji_frequency, calculate_heatmap_data, calculate_total_chars_per_game,
     calculate_reading_time_per_game, calculate_reading_speed_per_game,
@@ -32,6 +35,7 @@ def register_database_api_routes(app):
             sort_by = request.args.get('sort', 'relevance')
             page = int(request.args.get('page', 1))
             page_size = int(request.args.get('page_size', 20))
+            use_regex = request.args.get('use_regex', 'false').lower() == 'true'
             
             # Validate parameters
             if not query:
@@ -41,65 +45,129 @@ def register_database_api_routes(app):
                 page = 1
             if page_size < 1 or page_size > 100:
                 page_size = 20
-            
-            # Build the SQL query
-            base_query = f"SELECT * FROM {GameLinesTable._table} WHERE line_text LIKE ?"
-            params = [f'%{query}%']
-            
-            # Add game filter if specified
-            if game_filter:
-                base_query += " AND game_name = ?"
-                params.append(game_filter)
-            
-            # Add sorting
-            if sort_by == 'date_desc':
-                base_query += " ORDER BY timestamp DESC"
-            elif sort_by == 'date_asc':
-                base_query += " ORDER BY timestamp ASC"
-            elif sort_by == 'game_name':
-                base_query += " ORDER BY game_name, timestamp DESC"
-            else:  # relevance - could be enhanced with proper scoring
-                base_query += " ORDER BY timestamp DESC"
-            
-            # Get total count for pagination
-            count_query = f"SELECT COUNT(*) FROM {GameLinesTable._table} WHERE line_text LIKE ?"
-            count_params = [f'%{query}%']
-            if game_filter:
-                count_query += " AND game_name = ?"
-                count_params.append(game_filter)
-            
-            total_results = GameLinesTable._db.fetchone(count_query, count_params)[0]
-            
-            # Add pagination
-            offset = (page - 1) * page_size
-            base_query += f" LIMIT ? OFFSET ?"
-            params.extend([page_size, offset])
-            
-            # Execute search query
-            rows = GameLinesTable._db.fetchall(base_query, params)
-            
-            # Format results
-            results = []
-            for row in rows:
-                game_line = GameLinesTable.from_row(row)
-                if game_line:
-                    results.append({
-                        'id': game_line.id,
-                        'sentence': game_line.line_text or '',
-                        'game_name': game_line.game_name or 'Unknown Game',
-                        'timestamp': float(game_line.timestamp) if game_line.timestamp else 0,
-                        'translation': game_line.translation or None,
-                        'has_audio': bool(game_line.audio_path),
-                        'has_screenshot': bool(game_line.screenshot_path)
-                    })
-            
-            return jsonify({
-                'results': results,
-                'total': total_results,
-                'page': page,
-                'page_size': page_size,
-                'total_pages': (total_results + page_size - 1) // page_size
-            }), 200
+
+            if use_regex:
+                # Regex search: fetch all candidate rows, filter in Python
+                try:
+                    # Ensure query is a string
+                    if not isinstance(query, str):
+                        return jsonify({'error': 'Invalid query parameter type'}), 400
+                    
+                    all_lines = GameLinesTable.all()
+                    if game_filter:
+                        all_lines = [line for line in all_lines if line.game_name == game_filter]
+                    
+                    # Compile regex pattern with proper error handling
+                    try:
+                        pattern = re.compile(query, re.IGNORECASE)
+                    except re.error as regex_err:
+                        return jsonify({'error': f'Invalid regex pattern: {str(regex_err)}'}), 400
+                    
+                    # Filter lines using regex
+                    filtered_lines = []
+                    for line in all_lines:
+                        if line.line_text and isinstance(line.line_text, str):
+                            try:
+                                if pattern.search(line.line_text):
+                                    filtered_lines.append(line)
+                            except Exception as search_err:
+                                # Log but continue with other lines
+                                logger.warning(f"Regex search error on line {line.id}: {search_err}")
+                                continue
+                    
+                    # Sorting (default: timestamp DESC, or as specified)
+                    if sort_by == 'date_asc':
+                        filtered_lines.sort(key=lambda l: float(l.timestamp) if l.timestamp else 0)
+                    elif sort_by == 'game_name':
+                        filtered_lines.sort(key=lambda l: (l.game_name or '', -(float(l.timestamp) if l.timestamp else 0)))
+                    else:  # date_desc or relevance
+                        filtered_lines.sort(key=lambda l: -(float(l.timestamp) if l.timestamp else 0))
+                    
+                    total_results = len(filtered_lines)
+                    # Pagination
+                    start = (page - 1) * page_size
+                    end = start + page_size
+                    paged_lines = filtered_lines[start:end]
+                    results = []
+                    for line in paged_lines:
+                        results.append({
+                            'id': line.id,
+                            'sentence': line.line_text or '',
+                            'game_name': line.game_name or 'Unknown Game',
+                            'timestamp': float(line.timestamp) if line.timestamp else 0,
+                            'translation': line.translation or None,
+                            'has_audio': bool(getattr(line, 'audio_path', None)),
+                            'has_screenshot': bool(getattr(line, 'screenshot_path', None))
+                        })
+                    return jsonify({
+                        'results': results,
+                        'total': total_results,
+                        'page': page,
+                        'page_size': page_size,
+                        'total_pages': (total_results + page_size - 1) // page_size
+                    }), 200
+                except Exception as e:
+                    logger.error(f"Regex search failed: {e}")
+                    return jsonify({'error': f'Search failed: {str(e)}'}), 500
+            else:
+                # Build the SQL query
+                base_query = f"SELECT * FROM {GameLinesTable._table} WHERE line_text LIKE ?"
+                params = [f'%{query}%']
+                
+                # Add game filter if specified
+                if game_filter:
+                    base_query += " AND game_name = ?"
+                    params.append(game_filter)
+                
+                # Add sorting
+                if sort_by == 'date_desc':
+                    base_query += " ORDER BY timestamp DESC"
+                elif sort_by == 'date_asc':
+                    base_query += " ORDER BY timestamp ASC"
+                elif sort_by == 'game_name':
+                    base_query += " ORDER BY game_name, timestamp DESC"
+                else:  # relevance - could be enhanced with proper scoring
+                    base_query += " ORDER BY timestamp DESC"
+                
+                # Get total count for pagination
+                count_query = f"SELECT COUNT(*) FROM {GameLinesTable._table} WHERE line_text LIKE ?"
+                count_params = [f'%{query}%']
+                if game_filter:
+                    count_query += " AND game_name = ?"
+                    count_params.append(game_filter)
+                
+                total_results = GameLinesTable._db.fetchone(count_query, count_params)[0]
+                
+                # Add pagination
+                offset = (page - 1) * page_size
+                base_query += f" LIMIT ? OFFSET ?"
+                params.extend([page_size, offset])
+                
+                # Execute search query
+                rows = GameLinesTable._db.fetchall(base_query, params)
+                
+                # Format results
+                results = []
+                for row in rows:
+                    game_line = GameLinesTable.from_row(row)
+                    if game_line:
+                        results.append({
+                            'id': game_line.id,
+                            'sentence': game_line.line_text or '',
+                            'game_name': game_line.game_name or 'Unknown Game',
+                            'timestamp': float(game_line.timestamp) if game_line.timestamp else 0,
+                            'translation': game_line.translation or None,
+                            'has_audio': bool(game_line.audio_path),
+                            'has_screenshot': bool(game_line.screenshot_path)
+                        })
+                
+                return jsonify({
+                    'results': results,
+                    'total': total_results,
+                    'page': page,
+                    'page_size': page_size,
+                    'total_pages': (total_results + page_size - 1) // page_size
+                }), 200
             
         except ValueError as e:
             return jsonify({'error': 'Invalid pagination parameters'}), 400
@@ -226,14 +294,17 @@ def register_database_api_routes(app):
     @app.route('/api/settings', methods=['GET'])
     def api_get_settings():
         """
-        Get current AFK timer, session gap, and streak requirement settings.
+        Get current AFK timer, session gap, streak requirement, and goal settings.
         """
         try:
-            config = get_config()
+            config = get_stats_config()
             return jsonify({
-                'afk_timer_seconds': config.advanced.afk_timer_seconds,
-                'session_gap_seconds': config.advanced.session_gap_seconds,
-                'streak_requirement_hours': getattr(config.advanced, 'streak_requirement_hours', 1.0)
+                'afk_timer_seconds': config.afk_timer_seconds,
+                'session_gap_seconds': config.session_gap_seconds,
+                'streak_requirement_hours': config.streak_requirement_hours,
+                'reading_hours_target': config.reading_hours_target,
+                'character_count_target': config.character_count_target,
+                'games_target': config.games_target
             }), 200
         except Exception as e:
             logger.error(f"Error getting settings: {e}")
@@ -242,7 +313,7 @@ def register_database_api_routes(app):
     @app.route('/api/settings', methods=['POST'])
     def api_save_settings():
         """
-        Save/update AFK timer, session gap, and streak requirement settings.
+        Save/update AFK timer, session gap, streak requirement, and goal settings.
         """
         try:
             data = request.get_json()
@@ -253,6 +324,9 @@ def register_database_api_routes(app):
             afk_timer = data.get('afk_timer_seconds')
             session_gap = data.get('session_gap_seconds')
             streak_requirement = data.get('streak_requirement_hours')
+            reading_hours_target = data.get('reading_hours_target')
+            character_count_target = data.get('character_count_target')
+            games_target = data.get('games_target')
             
             # Validate input - only require the settings that are provided
             settings_to_update = {}
@@ -284,22 +358,54 @@ def register_database_api_routes(app):
                 except (ValueError, TypeError):
                     return jsonify({'error': 'Streak requirement must be a valid number'}), 400
             
+            if reading_hours_target is not None:
+                try:
+                    reading_hours_target = int(reading_hours_target)
+                    if reading_hours_target < 1 or reading_hours_target > 10000:
+                        return jsonify({'error': 'Reading hours target must be between 1 and 10,000 hours'}), 400
+                    settings_to_update['reading_hours_target'] = reading_hours_target
+                except (ValueError, TypeError):
+                    return jsonify({'error': 'Reading hours target must be a valid integer'}), 400
+            
+            if character_count_target is not None:
+                try:
+                    character_count_target = int(character_count_target)
+                    if character_count_target < 1000 or character_count_target > 1000000000:
+                        return jsonify({'error': 'Character count target must be between 1,000 and 1,000,000,000 characters'}), 400
+                    settings_to_update['character_count_target'] = character_count_target
+                except (ValueError, TypeError):
+                    return jsonify({'error': 'Character count target must be a valid integer'}), 400
+            
+            if games_target is not None:
+                try:
+                    games_target = int(games_target)
+                    if games_target < 1 or games_target > 1000:
+                        return jsonify({'error': 'Games target must be between 1 and 1,000'}), 400
+                    settings_to_update['games_target'] = games_target
+                except (ValueError, TypeError):
+                    return jsonify({'error': 'Games target must be a valid integer'}), 400
+            
             if not settings_to_update:
                 return jsonify({'error': 'No valid settings provided'}), 400
             
             # Update configuration
-            config = get_config()
+            config = get_stats_config()
             
             if 'afk_timer_seconds' in settings_to_update:
-                config.advanced.afk_timer_seconds = settings_to_update['afk_timer_seconds']
+                config.afk_timer_seconds = settings_to_update['afk_timer_seconds']
             if 'session_gap_seconds' in settings_to_update:
-                config.advanced.session_gap_seconds = settings_to_update['session_gap_seconds']
+                config.session_gap_seconds = settings_to_update['session_gap_seconds']
             if 'streak_requirement_hours' in settings_to_update:
-                setattr(config.advanced, 'streak_requirement_hours', settings_to_update['streak_requirement_hours'])
-            
-            # Save configuration
-            save_current_config(config)
-            
+                config.streak_requirement_hours = settings_to_update['streak_requirement_hours']
+            if 'reading_hours_target' in settings_to_update:
+                config.reading_hours_target = settings_to_update['reading_hours_target']
+            if 'character_count_target' in settings_to_update:
+                config.character_count_target = settings_to_update['character_count_target']
+            if 'games_target' in settings_to_update:
+                config.games_target = settings_to_update['games_target']
+            
+            save_stats_config(config)
+
             logger.info(f"Settings updated: {settings_to_update}")
             
             response_data = {'message': 'Settings saved successfully'}
@@ -685,6 +791,7 @@ def register_database_api_routes(app):
         Provides aggregated, cumulative stats for charting.
         Accepts optional 'year' parameter to filter heatmap data.
         """
+        punctionation_regex = regex.compile(r'[\p{P}\p{S}\p{Z}]')
         # Get optional year filter parameter
         filter_year = request.args.get('year', None)
         
@@ -697,13 +804,33 @@ def register_database_api_routes(app):
         # 2. Process data into daily totals for each game
         # Structure: daily_data[date_str][game_name] = {'lines': N, 'chars': N}
         daily_data = defaultdict(lambda: defaultdict(lambda: {'lines': 0, 'chars': 0}))
+        
+        
+        # start_time = time.perf_counter()
+        # for line in all_lines:
+        #     day_str = datetime.date.fromtimestamp(float(line.timestamp)).strftime('%Y-%m-%d')
+        #     game = line.game_name or "Unknown Game"
+        #     daily_data[day_str][game]['lines'] += 1
+        #     daily_data[day_str][game]['chars'] += len(line.line_text) if line.line_text else 0
+        # end_time = time.perf_counter()
+        # logger.info(f"Without Punctuation removal and daily aggregation took {end_time - start_time:.4f} seconds for {len(all_lines)} lines")
 
+        # start_time = time.perf_counter()
+        wrong_instance_found = False
         for line in all_lines:
             day_str = datetime.date.fromtimestamp(float(line.timestamp)).strftime('%Y-%m-%d')
             game = line.game_name or "Unknown Game"
-            
+            # Remove punctuation and symbols from line text before counting characters
+            clean_text = punctionation_regex.sub('', str(line.line_text)) if line.line_text else ''
+            if not isinstance(clean_text, str) and not wrong_instance_found:
+                logger.info(f"Non-string line_text encountered: {clean_text} (type: {type(clean_text)})")
+                wrong_instance_found = True
+
+            line.line_text = clean_text  # Update line text to cleaned version for future use
             daily_data[day_str][game]['lines'] += 1
-            daily_data[day_str][game]['chars'] += len(line.line_text) if line.line_text else 0
+            daily_data[day_str][game]['chars'] += len(clean_text)
+        # end_time = time.perf_counter()
+        # logger.info(f"With Punctuation removal and daily aggregation took {end_time - start_time:.4f} seconds for {len(all_lines)} lines")
 
         # 3. Create cumulative datasets for Chart.js
         sorted_days = sorted(daily_data.keys())