GameSentenceMiner 2.19.16__py3-none-any.whl → 2.20.0__py3-none-any.whl

GameSentenceMiner/web/static/js/regex-patterns.js

@@ -0,0 +1,100 @@
+// Shared regex patterns for search and deletion functionality
+// This module provides preset regex patterns and helper functions
+
+/**
+ * Preset regex patterns with descriptions
+ * Each pattern includes:
+ * - value: unique identifier
+ * - label: user-friendly description
+ * - pattern: the actual regex pattern
+ */
+export const PRESET_PATTERNS = {
+    'lines_over_50': {
+        label: 'Lines over 50 characters',
+        pattern: '.{51,}'
+    },
+    'lines_over_100': {
+        label: 'Lines over 100 characters',
+        pattern: '.{101,}'
+    },
+    'non_japanese': {
+        label: 'Non-Japanese text',
+        pattern: '^[^\u3040-\u309F\u30A0-\u30FF\u4E00-\u9FAF]*$'
+    },
+    'ascii_only': {
+        label: 'ASCII-only lines',
+        pattern: '^[\x00-\x7F]*$'
+    },
+    'empty_lines': {
+        label: 'Empty or whitespace-only lines',
+        pattern: '^\s*$'
+    },
+    'numbers_only': {
+        label: 'Lines with numbers only',
+        pattern: '^\d+$'
+    },
+    'single_char': {
+        label: 'Single character lines',
+        pattern: '^.{1}$'
+    },
+    'repeated_chars': {
+        label: 'Lines with repeated characters (3+ times)',
+        pattern: '(.)\\1{2,}'
+    },
+    'everything': {
+        label: 'Everything',
+        pattern: '.*'
+    }
+};
+
+/**
+ * Get pattern by key
+ * @param {string} key - Pattern key
+ * @returns {string|null} - Pattern string or null if not found
+ */
+export function getPattern(key) {
+    return PRESET_PATTERNS[key]?.pattern || null;
+}
+
+/**
+ * Get all pattern options for dropdown
+ * @returns {Array} - Array of {value, label} objects
+ */
+export function getPatternOptions() {
+    return Object.entries(PRESET_PATTERNS).map(([key, data]) => ({
+        value: key,
+        label: data.label
+    }));
+}
+
+/**
+ * Validate regex pattern
+ * @param {string} pattern - Regex pattern to validate
+ * @returns {Object} - {valid: boolean, error: string|null}
+ */
+export function validateRegex(pattern) {
+    try {
+        new RegExp(pattern);
+        return { valid: true, error: null };
+    } catch (e) {
+        return { valid: false, error: e.message };
+    }
+}
+
+/**
+ * Apply regex pattern to text
+ * @param {string} text - Text to test
+ * @param {string} pattern - Regex pattern
+ * @param {boolean} caseSensitive - Case sensitivity flag
+ * @returns {boolean} - True if pattern matches
+ */
+export function testPattern(text, pattern, caseSensitive = false) {
+    try {
+        const flags = caseSensitive ? 'g' : 'gi';
+        const regex = new RegExp(pattern, flags);
+        return regex.test(text);
+    } catch (e) {
+        console.error('Regex test error:', e);
+        return false;
+    }
+}

GameSentenceMiner/web/static/js/search.js

@@ -3,6 +3,8 @@ class SentenceSearchApp {
         this.searchInput = document.getElementById('searchInput');
         this.gameFilter = document.getElementById('gameFilter');
         this.sortFilter = document.getElementById('sortFilter');
+        this.fromDateFilter = document.getElementById('searchFromDate');
+        this.toDateFilter = document.getElementById('searchToDate');
         this.searchResults = document.getElementById('searchResults');
         this.loadingIndicator = document.getElementById('loadingIndicator');
         this.noResults = document.getElementById('noResults');
@@ -10,10 +12,27 @@ class SentenceSearchApp {
         this.errorMessage = document.getElementById('errorMessage');
         this.searchStats = document.getElementById('searchStats');
         this.searchTime = document.getElementById('searchTime');
-        this.regexCheckbox = document.getElementById('regexCheckbox');
+        
+        // Regex component elements
+        this.regexPresetSelect = document.querySelector('.regex-preset-select');
+        this.regexCustomInput = document.querySelector('.regex-custom-input');
+        this.regexCaseCheckbox = document.querySelector('.regex-case-checkbox');
+        this.regexModeCheckbox = document.querySelector('.regex-mode-checkbox');
+        
+        // Duplicate detection elements
+        this.toggleDuplicateBtn = document.getElementById('toggleDuplicateDetection');
+        this.duplicateSection = document.getElementById('duplicateDetectionSection');
+        this.duplicateTimeWindow = document.getElementById('duplicateTimeWindow');
+        this.duplicateIgnoreTimeWindow = document.getElementById('duplicateIgnoreTimeWindow');
+        this.duplicateCaseSensitive = document.getElementById('duplicateCaseSensitive');
+        this.searchDuplicatesBtn = document.getElementById('searchDuplicatesBtn');
+        this.duplicateTimeWindowGroup = document.getElementById('duplicateTimeWindowGroup');
+        
         this.deleteLinesBtn = document.getElementById('deleteLinesBtn');
         this.selectAllBtn = document.getElementById('selectAllBtn');
         this.pageSizeFilter = document.getElementById('pageSizeFilter');
+        this.toggleAdvancedBtn = document.getElementById('toggleAdvancedSearch');
+        this.advancedSearchSection = document.getElementById('advancedSearchSection');
         
         this.currentPage = 1;
         this.pageSize = 20;
@@ -21,6 +40,7 @@ class SentenceSearchApp {
         this.currentQuery = '';
         this.totalResults = 0;
         this.currentUseRegex = false;
+        this.isDuplicateSearch = false;
         this.initialize();
     }
 
@@ -56,6 +76,8 @@ class SentenceSearchApp {
         this.gameFilter.addEventListener('change', () => this.performSearch());
         this.sortFilter.addEventListener('change', () => this.performSearch());
         
+        // Date range filters do NOT auto-trigger search - user must click Search button
+        
         if (this.pageSizeFilter) {
             this.pageSizeFilter.addEventListener('change', () => {
                 this.pageSize = parseInt(this.pageSizeFilter.value);
@@ -76,8 +98,24 @@ class SentenceSearchApp {
             this.performSearch();
         });
 
-        if (this.regexCheckbox) {
-            this.regexCheckbox.addEventListener('change', () => {
+        // Regex component event listeners
+        if (this.regexCustomInput) {
+            this.regexCustomInput.addEventListener('input', () => {
+                clearTimeout(this.searchTimeout);
+                this.searchTimeout = setTimeout(() => {
+                    this.performSearch();
+                }, 300);
+            });
+        }
+        
+        if (this.regexModeCheckbox) {
+            this.regexModeCheckbox.addEventListener('change', () => {
+                this.performSearch();
+            });
+        }
+        
+        if (this.regexCaseCheckbox) {
+            this.regexCaseCheckbox.addEventListener('change', () => {
                 this.performSearch();
             });
         }
@@ -93,6 +131,53 @@ class SentenceSearchApp {
                 this.toggleSelectAll();
             });
         }
+
+        if (this.toggleAdvancedBtn) {
+            this.toggleAdvancedBtn.addEventListener('click', () => {
+                this.toggleAdvancedSearch();
+            });
+        }
+        
+        // Duplicate detection event listeners
+        if (this.toggleDuplicateBtn) {
+            this.toggleDuplicateBtn.addEventListener('click', () => {
+                this.toggleDuplicateDetection();
+            });
+        }
+        
+        if (this.duplicateIgnoreTimeWindow) {
+            this.duplicateIgnoreTimeWindow.addEventListener('change', () => {
+                this.toggleDuplicateTimeWindow();
+            });
+        }
+        
+        if (this.searchDuplicatesBtn) {
+            this.searchDuplicatesBtn.addEventListener('click', () => {
+                this.searchForDuplicates();
+            });
+        }
+        
+        // Manual search button for date filtering
+        const manualSearchBtn = document.getElementById('manualSearchBtn');
+        if (manualSearchBtn) {
+            manualSearchBtn.addEventListener('click', () => {
+                this.performSearch();
+            });
+        }
+    }
+
+    toggleAdvancedSearch() {
+        if (!this.advancedSearchSection || !this.toggleAdvancedBtn) return;
+        
+        const isHidden = this.advancedSearchSection.style.display === 'none';
+        
+        if (isHidden) {
+            this.advancedSearchSection.style.display = 'block';
+            this.toggleAdvancedBtn.innerHTML = '<span id="toggleIcon">▲</span> Hide Advanced Search';
+        } else {
+            this.advancedSearchSection.style.display = 'none';
+            this.toggleAdvancedBtn.innerHTML = '<span id="toggleIcon">▼</span> Show Advanced Search';
+        }
     }
     
     async loadGamesList() {
@@ -120,25 +205,36 @@ class SentenceSearchApp {
         const query = this.searchInput.value.trim();
         const gameFilter = this.gameFilter.value;
         const sortBy = this.sortFilter.value;
-        const useRegex = this.regexCheckbox && this.regexCheckbox.checked;
-
-        if (query !== this.currentQuery || useRegex !== this.currentUseRegex) {
-            this.currentPage = 1;
+        const fromDate = this.fromDateFilter ? this.fromDateFilter.value : '';
+        const toDate = this.toDateFilter ? this.toDateFilter.value : '';
+        
+        // Get regex settings from component
+        const customRegex = this.regexCustomInput ? this.regexCustomInput.value.trim() : '';
+        const useRegex = this.regexModeCheckbox ? this.regexModeCheckbox.checked : false;
+        const caseSensitive = this.regexCaseCheckbox ? this.regexCaseCheckbox.checked : false;
+        
+        // Use custom regex if provided and regex mode is enabled, otherwise use search query
+        // If search is empty, default to .* regex pattern
+        let searchPattern = (useRegex && customRegex) ? customRegex : query;
+        let effectiveUseRegex = useRegex;
+        
+        if (!searchPattern && !query) {
+            searchPattern = '.*';
+            effectiveUseRegex = true;  // Force regex mode for .* pattern
         }
-        this.currentQuery = query;
-        this.currentUseRegex = useRegex;
 
-        if (!query) {
-            this.showEmptyState();
-            return;
+        if (searchPattern !== this.currentQuery || effectiveUseRegex !== this.currentUseRegex) {
+            this.currentPage = 1;
         }
+        this.currentQuery = searchPattern;
+        this.currentUseRegex = effectiveUseRegex;
 
         this.showLoadingState();
         const startTime = Date.now();
 
         try {
             const params = new URLSearchParams({
-                q: query,
+                q: searchPattern,
                 page: this.currentPage,
                 page_size: this.pageSize,
                 sort: sortBy
@@ -147,9 +243,18 @@ class SentenceSearchApp {
             if (gameFilter) {
                 params.append('game', gameFilter);
             }
-            if (useRegex) {
+            if (fromDate) {
+                params.append('from_date', fromDate);
+            }
+            if (toDate) {
+                params.append('to_date', toDate);
+            }
+            if (effectiveUseRegex) {
                 params.append('use_regex', 'true');
             }
+            if (caseSensitive) {
+                params.append('case_sensitive', 'true');
+            }
 
             const response = await fetch(`/api/search-sentences?${params}`);
             const data = await response.json();
@@ -248,12 +353,15 @@ class SentenceSearchApp {
     highlightSearchTerms(text, query) {
         if (!query) return escapeHtml(text);
 
-        const useRegex = this.regexCheckbox && this.regexCheckbox.checked;
+        const useRegex = this.regexModeCheckbox ? this.regexModeCheckbox.checked : false;
+        const customRegex = this.regexCustomInput ? this.regexCustomInput.value.trim() : '';
+        const caseSensitive = this.regexCaseCheckbox ? this.regexCaseCheckbox.checked : false;
         const escapedText = escapeHtml(text);
 
-        if (useRegex) {
+        if (useRegex && customRegex) {
             try {
-                const pattern = new RegExp(query, 'gi');
+                const flags = caseSensitive ? 'g' : 'gi';
+                const pattern = new RegExp(customRegex, flags);
                 return escapedText.replace(pattern, '<span class="search-highlight">$&</span>');
             } catch (e) {
                 return escapedText;
@@ -262,7 +370,8 @@ class SentenceSearchApp {
             const searchTerms = query.split(' ').filter(term => term.length > 0);
             let result = escapedText;
             searchTerms.forEach(term => {
-                const regex = new RegExp(`(${escapeRegex(term)})`, 'gi');
+                const flags = caseSensitive ? 'g' : 'gi';
+                const regex = new RegExp(`(${escapeRegex(term)})`, flags);
                 result = result.replace(regex, '<span class="search-highlight">$1</span>');
             });
             return result;
@@ -436,6 +545,94 @@ class SentenceSearchApp {
         }
     }
 
+    toggleDuplicateDetection() {
+        if (!this.duplicateSection || !this.toggleDuplicateBtn) return;
+        
+        const isHidden = this.duplicateSection.style.display === 'none';
+        const icon = document.getElementById('duplicateToggleIcon');
+        
+        if (isHidden) {
+            this.duplicateSection.style.display = 'block';
+            if (icon) icon.textContent = '▲';
+        } else {
+            this.duplicateSection.style.display = 'none';
+            if (icon) icon.textContent = '▼';
+        }
+    }
+    
+    toggleDuplicateTimeWindow() {
+        if (!this.duplicateIgnoreTimeWindow || !this.duplicateTimeWindowGroup) return;
+        
+        const isIgnored = this.duplicateIgnoreTimeWindow.checked;
+        
+        if (isIgnored) {
+            this.duplicateTimeWindowGroup.style.opacity = '0.5';
+            this.duplicateTimeWindowGroup.style.pointerEvents = 'none';
+            if (this.duplicateTimeWindow) {
+                this.duplicateTimeWindow.disabled = true;
+            }
+        } else {
+            this.duplicateTimeWindowGroup.style.opacity = '1';
+            this.duplicateTimeWindowGroup.style.pointerEvents = 'auto';
+            if (this.duplicateTimeWindow) {
+                this.duplicateTimeWindow.disabled = false;
+            }
+        }
+    }
+    
+    async searchForDuplicates() {
+        const gameFilter = this.gameFilter.value;
+        const timeWindow = parseInt(this.duplicateTimeWindow.value);
+        const ignoreTimeWindow = this.duplicateIgnoreTimeWindow.checked;
+        const caseSensitive = this.duplicateCaseSensitive.checked;
+        
+        // Validate input
+        if (!ignoreTimeWindow && (isNaN(timeWindow) || timeWindow < 1)) {
+            this.showErrorState('Time window must be at least 1 minute');
+            return;
+        }
+        
+        this.showLoadingState();
+        this.isDuplicateSearch = true;
+        const startTime = Date.now();
+        
+        try {
+            const requestData = {
+                game: gameFilter,
+                time_window_minutes: timeWindow,
+                ignore_time_window: ignoreTimeWindow,
+                case_sensitive: caseSensitive
+            };
+            
+            const response = await fetch('/api/search-duplicates', {
+                method: 'POST',
+                headers: { 'Content-Type': 'application/json' },
+                body: JSON.stringify(requestData)
+            });
+            
+            const data = await response.json();
+            const searchTime = Date.now() - startTime;
+            
+            if (!response.ok) {
+                throw new Error(data.error || 'Duplicate search failed');
+            }
+            
+            // Display results using existing display method
+            this.displayResults(data, searchTime);
+            
+            // Update stats text to indicate duplicate search
+            if (data.total > 0) {
+                const modeText = ignoreTimeWindow ? 'across entire game' : `within ${timeWindow} minute window`;
+                const gameText = gameFilter ? ` in ${gameFilter}` : '';
+                this.searchStats.textContent = `Found ${data.total.toLocaleString()} duplicate sentences ${modeText}${gameText}`;
+            }
+            
+        } catch (error) {
+            this.showErrorState(error.message);
+            this.isDuplicateSearch = false;
+        }
+    }
+
     showMessage(title, message) {
         document.getElementById('messageModalTitle').textContent = title;
         document.getElementById('messageModalText').textContent = message;

GameSentenceMiner/web/static/js/shared.js

@@ -517,6 +517,62 @@ function escapeRegex(string) {
     return string.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
 }
 
+function safeJoinArray(arr, separator = ', ') {
+    /**
+     * Safely join an array with proper type checking and fallbacks.
+     * Handles various data types that might be returned from API responses.
+     *
+     * @param {*} arr - The value to join (should be an array, but handles other types)
+     * @param {string} separator - The separator to use for joining
+     * @returns {string} - The joined string or appropriate fallback
+     */
+    if (!arr) {
+        return '';
+    }
+    
+    if (Array.isArray(arr)) {
+        return arr.join(separator);
+    }
+    
+    if (typeof arr === 'string') {
+        return arr;
+    }
+    
+    // Handle other types by converting to string
+    return String(arr);
+}
+
+function logApiResponse(operation, response, result) {
+    /**
+     * Log API response details for debugging purposes.
+     *
+     * @param {string} operation - The operation being performed
+     * @param {Response} response - The fetch response object
+     * @param {*} result - The parsed JSON result
+     */
+    console.group(`🔍 API Response Debug: ${operation}`);
+    console.log('Response status:', response.status, response.statusText);
+    console.log('Response OK:', response.ok);
+    console.log('Result object:', result);
+    
+    // Print FULL API response as formatted JSON
+    console.log('%c📋 FULL API RESPONSE (JSON):', 'color: #00ff00; font-weight: bold; font-size: 14px;');
+    console.log(JSON.stringify(result, null, 2));
+    
+    if (result && typeof result === 'object') {
+        Object.keys(result).forEach(key => {
+            const value = result[key];
+            console.log(`${key}:`, {
+                value,
+                type: typeof value,
+                isArray: Array.isArray(value),
+                length: Array.isArray(value) ? value.length : 'N/A'
+            });
+        });
+    }
+    console.groupEnd();
+}
+
 // Screenshot functionality
 function initializeScreenshotButton() {
     const screenshotButton = document.getElementById('screenshotToggle');
@@ -525,62 +581,160 @@ function initializeScreenshotButton() {
         return; // Screenshot button not available on this page
     }
     
-    screenshotButton.addEventListener('click', takeScreenshot);
+    screenshotButton.addEventListener('click', exportPageToPDF);
+}
+
+// Lazy-load libraries
+let html2canvasLoaded = false;
+let html2canvasLoading = false;
+let jsPDFLoaded = false;
+let jsPDFLoading = false;
+
+// Lazy-load html2canvas
+async function loadHtml2Canvas() {
+    if (html2canvasLoaded) return true;
+    if (html2canvasLoading) {
+        return new Promise(resolve => {
+            const check = setInterval(() => {
+                if (html2canvasLoaded) {
+                    clearInterval(check);
+                    resolve(true);
+                }
+            }, 100);
+        });
+    }
+
+    html2canvasLoading = true;
+    return new Promise((resolve, reject) => {
+        const script = document.createElement('script');
+        script.src = 'https://cdn.jsdelivr.net/npm/html2canvas@1.4.1/dist/html2canvas.min.js';
+        script.onload = () => {
+            html2canvasLoaded = true;
+            html2canvasLoading = false;
+            resolve(true);
+        };
+        script.onerror = () => {
+            html2canvasLoading = false;
+            reject(new Error('Failed to load html2canvas'));
+        };
+        document.head.appendChild(script);
+    });
 }
 
-async function takeScreenshot() {
+// Lazy-load jsPDF
+async function loadJsPDF() {
+    if (jsPDFLoaded) return true;
+    if (jsPDFLoading) {
+        return new Promise(resolve => {
+            const check = setInterval(() => {
+                if (jsPDFLoaded) {
+                    clearInterval(check);
+                    resolve(true);
+                }
+            }, 100);
+        });
+    }
+
+    jsPDFLoading = true;
+    return new Promise((resolve, reject) => {
+        const script = document.createElement('script');
+        script.src = 'https://cdnjs.cloudflare.com/ajax/libs/jspdf/2.5.1/jspdf.umd.min.js';
+        script.onload = () => {
+            jsPDFLoaded = true;
+            jsPDFLoading = false;
+            resolve(true);
+        };
+        script.onerror = () => {
+            jsPDFLoading = false;
+            reject(new Error('Failed to load jsPDF'));
+        };
+        document.head.appendChild(script);
+    });
+}
+
+// Capture page and export as PDF
+async function exportPageToPDF() {
     try {
-        // Check if html2canvas is available
+        console.log('Starting PDF export...');
+        
+        // Load libraries
         if (typeof html2canvas === 'undefined') {
-            console.error('html2canvas library not loaded');
-            return;
+            console.log('Loading html2canvas...');
+            await loadHtml2Canvas();
         }
-        
-        // Generate timestamp for filename
+        if (typeof window.jspdf === 'undefined') {
+            console.log('Loading jsPDF...');
+            await loadJsPDF();
+        }
+
+        const { jsPDF } = window.jspdf;
+
+        // Create timestamped filename
         const now = new Date();
-        const timestamp = now.getFullYear() + '-' +
-                         String(now.getMonth() + 1).padStart(2, '0') + '-' +
-                         String(now.getDate()).padStart(2, '0') + '_' +
-                         String(now.getHours()).padStart(2, '0') + '-' +
-                         String(now.getMinutes()).padStart(2, '0') + '-' +
-                         String(now.getSeconds()).padStart(2, '0');
-        
-        const filename = `screenshot_${timestamp}.png`;
-        
-        // Capture the entire page
+        const timestamp =
+            now.getFullYear() + '-' +
+            String(now.getMonth() + 1).padStart(2, '0') + '-' +
+            String(now.getDate()).padStart(2, '0') + '_' +
+            String(now.getHours()).padStart(2, '0') + '-' +
+            String(now.getMinutes()).padStart(2, '0') + '-' +
+            String(now.getSeconds()).padStart(2, '0');
+        const filename = `GSM_STATS_${timestamp}.pdf`;
+
+        console.log('Capturing page screenshot...');
+        // Take a screenshot of the full page with high quality for sharp text
         const canvas = await html2canvas(document.body, {
             useCORS: true,
             allowTaint: true,
-            scale: 1,
+            scale: 2.5,  // Higher scale for sharper text (2.5x resolution)
             scrollX: 0,
             scrollY: 0,
             width: document.body.scrollWidth,
-            height: document.body.scrollHeight
+            height: document.body.scrollHeight,
+            logging: false,
+            imageTimeout: 0,
+            removeContainer: true
         });
-        
-        // Convert canvas to blob
-        canvas.toBlob(function(blob) {
-            // Create download link
-            const link = document.createElement('a');
-            link.download = filename;
-            link.href = URL.createObjectURL(blob);
-            
-            // Trigger download
-            document.body.appendChild(link);
-            link.click();
-            document.body.removeChild(link);
-            
-            // Clean up the URL object after a short delay to avoid race condition
-            setTimeout(function() {
-                URL.revokeObjectURL(link.href);
-            }, 100);
-        }, 'image/png');
-        
+
+        console.log('Converting to image...');
+        // Use JPEG with high quality for better file size vs quality balance
+        const imgData = canvas.toDataURL('image/jpeg', 0.80);
+
+        console.log('Creating PDF...');
+        // PDF setup (A4)
+        const pdf = new jsPDF('p', 'pt', 'a4');
+        const pageWidth = pdf.internal.pageSize.getWidth();
+        const pageHeight = pdf.internal.pageSize.getHeight();
+
+        const imgWidth = pageWidth;
+        const imgHeight = (canvas.height * imgWidth) / canvas.width;
+
+        let heightLeft = imgHeight;
+        let position = 0;
+
+        // Add first page with SLOW compression for better quality
+        pdf.addImage(imgData, 'JPEG', 0, position, imgWidth, imgHeight, undefined, 'SLOW');
+        heightLeft -= pageHeight;
+
+        // Add more pages if needed
+        while (heightLeft > 0) {
+            position = heightLeft - imgHeight;
+            pdf.addPage();
+            pdf.addImage(imgData, 'JPEG', 0, position, imgWidth, imgHeight, undefined, 'SLOW');
+            heightLeft -= pageHeight;
+        }
+
+        console.log('Saving PDF...');
+        // Download
+        pdf.save(filename);
+        console.log('PDF export complete!');
+
     } catch (error) {
-        console.error('Screenshot failed:', error);
+        console.error('PDF export failed:', error);
+        alert('Failed to export PDF: ' + error.message);
     }
 }
 
+
 // Initialize shared functionality when DOM loads
 document.addEventListener('DOMContentLoaded', function() {
     // Initialize theme toggle