supervertaler 1.9.172__py3-none-any.whl → 1.9.173__py3-none-any.whl

Supervertaler.py

@@ -5598,7 +5598,7 @@ class PreTranslationWorker(QThread):
                     match = matches[0]
                     match_pct = match.get('match_pct', 0)
                     print(f"🔍 TM PRE-TRANSLATE: Best match pct: {match_pct}")
-                    if match_pct >= 70:  # Accept matches 70% and above
+                    if match_pct >= 75:  # Accept matches 75% and above
                         return match.get('target', '')
                 return None
         except Exception as e:
@@ -41148,25 +41148,41 @@ OUTPUT ONLY THE SEGMENT MARKERS. DO NOT ADD EXPLANATIONS BEFORE OR AFTER."""
             self.log(f"📖 Pre-translate from TM: Using activated TMs: {tm_ids}")
             
             # Create progress dialog for TM pre-translation
+            import time
+            start_time = time.time()
+            total_segments = len(segments_needing_translation)
+
             progress = QProgressDialog(
-                f"Pre-translating {len(segments_needing_translation)} segments from TM...",
-                "Cancel", 0, len(segments_needing_translation), self
+                f"Pre-translating {total_segments} segments from TM...",
+                "Cancel", 0, total_segments, self
             )
-            progress.setWindowTitle("TM Pre-Translation")
+            progress.setWindowTitle("🔍 TM Pre-Translation")
             progress.setWindowModality(Qt.WindowModality.WindowModal)
             progress.setMinimumDuration(0)  # Show immediately
+            progress.setMinimumWidth(450)  # Wider dialog for more info
             progress.show()
             QApplication.processEvents()
-            
+
             success_count = 0
             no_match_count = 0
-            
+
             for idx, (row_index, segment) in enumerate(segments_needing_translation):
                 if progress.wasCanceled():
                     break
-                
+
                 progress.setValue(idx)
-                progress.setLabelText(f"Searching TM for segment {idx + 1}/{len(segments_needing_translation)}...")
+
+                # Build informative progress label
+                elapsed = time.time() - start_time
+                elapsed_str = f"{int(elapsed // 60)}:{int(elapsed % 60):02d}"
+                source_preview = segment.source[:50] + "..." if len(segment.source) > 50 else segment.source
+                label_text = (
+                    f"Searching TM for segment {idx + 1} of {total_segments}...\n\n"
+                    f"Current: \"{source_preview}\"\n"
+                    f"Matches found: {success_count}  |  Elapsed: {elapsed_str}\n\n"
+                    f"ℹ️ This may take a while for large documents."
+                )
+                progress.setLabelText(label_text)
                 QApplication.processEvents()
                 
                 try:
@@ -41184,7 +41200,7 @@ OUTPUT ONLY THE SEGMENT MARKERS. DO NOT ADD EXPLANATIONS BEFORE OR AFTER."""
                         else:
                             no_match_count += 1
                     else:
-                        # Fuzzy matching enabled - get best match ≥70%
+                        # Fuzzy matching enabled - get best match ≥75%
                         matches = self.tm_database.search_all(
                             segment.source,
                             tm_ids=tm_ids,
@@ -41194,7 +41210,7 @@ OUTPUT ONLY THE SEGMENT MARKERS. DO NOT ADD EXPLANATIONS BEFORE OR AFTER."""
                         if matches and len(matches) > 0:
                             best_match = matches[0]
                             match_pct = best_match.get('match_pct', 0)
-                            if match_pct >= 70:
+                            if match_pct >= 75:
                                 segment.target = best_match.get('target', '')
                                 segment.status = "Translated"
                                 success_count += 1
@@ -41562,7 +41578,7 @@ OUTPUT ONLY THE SEGMENT MARKERS. DO NOT ADD EXPLANATIONS BEFORE OR AFTER."""
                             match_pct = match.get('match_pct', 0)
                             tm_match = match.get('target', '')
                             
-                            if match_pct >= 70:  # Accept matches 70% and above
+                            if match_pct >= 75:  # Accept matches 75% and above
                                 segment.target = tm_match
                                 segment.status = "translated" if match_pct == 100 else "pre-translated"
                                 translated_count += 1

modules/database_manager.py

@@ -17,12 +17,38 @@ import sqlite3
 import os
 import json
 import hashlib
+import unicodedata
+import re
 from datetime import datetime
 from typing import List, Dict, Optional, Tuple
 from pathlib import Path
 from difflib import SequenceMatcher
 
 
+def _normalize_for_matching(text: str) -> str:
+    """Normalize text for exact matching.
+
+    Handles invisible differences that would cause exact match to fail:
+    - Unicode normalization (NFC)
+    - Multiple whitespace -> single space
+    - Leading/trailing whitespace
+    - Non-breaking spaces -> regular spaces
+    """
+    if not text:
+        return ""
+    # Unicode normalize (NFC form)
+    text = unicodedata.normalize('NFC', text)
+    # Convert non-breaking spaces and other whitespace to regular space
+    text = text.replace('\u00a0', ' ')  # NBSP
+    text = text.replace('\u2007', ' ')  # Figure space
+    text = text.replace('\u202f', ' ')  # Narrow NBSP
+    # Collapse multiple whitespace to single space
+    text = re.sub(r'\s+', ' ', text)
+    # Strip leading/trailing whitespace
+    text = text.strip()
+    return text
+
+
 class DatabaseManager:
     """Manages SQLite database for translation resources"""
     
@@ -655,17 +681,19 @@ class DatabaseManager:
     # TRANSLATION MEMORY METHODS
     # ============================================
     
-    def add_translation_unit(self, source: str, target: str, source_lang: str, 
+    def add_translation_unit(self, source: str, target: str, source_lang: str,
                             target_lang: str, tm_id: str = 'project',
                             project_id: str = None, context_before: str = None,
                             context_after: str = None, notes: str = None) -> int:
         """
         Add translation unit to database
-        
+
         Returns: ID of inserted/updated entry
         """
-        # Generate hash for fast exact matching
-        source_hash = hashlib.md5(source.encode('utf-8')).hexdigest()
+        # Generate hash from NORMALIZED source for consistent exact matching
+        # This handles invisible differences like Unicode normalization, whitespace variations
+        normalized_source = _normalize_for_matching(source)
+        source_hash = hashlib.md5(normalized_source.encode('utf-8')).hexdigest()
         
         try:
             self.cursor.execute("""
@@ -687,33 +715,38 @@ class DatabaseManager:
             return None
     
     def get_exact_match(self, source: str, tm_ids: List[str] = None,
-                       source_lang: str = None, target_lang: str = None, 
+                       source_lang: str = None, target_lang: str = None,
                        bidirectional: bool = True) -> Optional[Dict]:
         """
         Get exact match from TM
-        
+
         Args:
             source: Source text to match
             tm_ids: List of TM IDs to search (None = all)
             source_lang: Filter by source language (base code matching: 'en' matches 'en-US', 'en-GB', etc.)
             target_lang: Filter by target language (base code matching)
             bidirectional: If True, search both directions (nl→en AND en→nl)
-        
+
         Returns: Dictionary with match data or None
         """
         from modules.tmx_generator import get_base_lang_code
-        
+
+        # Try both normalized and non-normalized hashes for backward compatibility
+        # This handles invisible differences like Unicode normalization, whitespace variations
         source_hash = hashlib.md5(source.encode('utf-8')).hexdigest()
-        
+        normalized_source = _normalize_for_matching(source)
+        normalized_hash = hashlib.md5(normalized_source.encode('utf-8')).hexdigest()
+
         # Get base language codes for comparison
         src_base = get_base_lang_code(source_lang) if source_lang else None
         tgt_base = get_base_lang_code(target_lang) if target_lang else None
-        
+
+        # Search using both original hash and normalized hash
         query = """
-            SELECT * FROM translation_units 
-            WHERE source_hash = ? AND source_text = ?
+            SELECT * FROM translation_units
+            WHERE (source_hash = ? OR source_hash = ?)
         """
-        params = [source_hash, source]
+        params = [source_hash, normalized_hash]
         
         if tm_ids:
             placeholders = ','.join('?' * len(tm_ids))

modules/translation_memory.py

@@ -123,8 +123,8 @@ class TMDatabase:
         if source_lang and target_lang:
             self.set_tm_languages(source_lang, target_lang)
         
-        # Global fuzzy threshold (70% minimum similarity for fuzzy matches)
-        self.fuzzy_threshold = 0.7
+        # Global fuzzy threshold (75% minimum similarity for fuzzy matches)
+        self.fuzzy_threshold = 0.75
         
         # TM metadata cache (populated from database as needed)
         # Note: Legacy 'project' and 'big_mama' TMs are no longer used.

{supervertaler-1.9.172.dist-info → supervertaler-1.9.173.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: supervertaler
-Version: 1.9.172
+Version: 1.9.173
 Summary: Professional AI-enhanced translation workbench with multi-LLM support, glossary system, TM, spellcheck, voice commands, and PyQt6 interface. Batteries included (core).
 Home-page: https://supervertaler.com
 Author: Michael Beijer
@@ -71,7 +71,7 @@ Dynamic: home-page
 Dynamic: license-file
 Dynamic: requires-python
 
-# 🚀 Supervertaler v1.9.172
+# 🚀 Supervertaler v1.9.173
 
 [![PyPI version](https://badge.fury.io/py/supervertaler.svg)](https://pypi.org/project/Supervertaler/)
 [![Python 3.10+](https://img.shields.io/badge/python-3.10+-blue.svg)](https://www.python.org/downloads/)
@@ -80,7 +80,13 @@ Dynamic: requires-python
 AI-enhanced CAT tool with multi-LLM support (GPT-4, Claude, Gemini, Ollama), innovative Superlookup concordance system offering access to multiple terminology sources (TMs, glossaries, web resources, etc.), and seamless CAT tool integration (memoQ, Trados, CafeTran, Phrase).
 
 
-**Current Version:** v1.9.172 (January 28, 2026)
+**Current Version:** v1.9.173 (January 28, 2026)
+
+### IMPROVED in v1.9.173 - 🎯 Smarter TM Pre-Translation
+
+- **Smarter TM Exact Matching**: Exact matches now use text normalization, so matches are found even with whitespace/Unicode differences.
+- **Improved Pre-Translation Dialog**: Shows current segment, match count, elapsed time, and patience message for large jobs.
+- **Higher Fuzzy Threshold**: Raised minimum fuzzy match threshold from 70% to 75% for better quality.
 
 ### FIXED in v1.9.172 - 🐛 Fresh Projects Start Clean
 

{supervertaler-1.9.172.dist-info → supervertaler-1.9.173.dist-info}/RECORD

@@ -1,4 +1,4 @@
-Supervertaler.py,sha256=QN11SrXGKNdKhXFRwgkumm6IRI6Q9vg_S2yj2Qbd3K4,2286283
+Supervertaler.py,sha256=Hzzf8r4zCdvk8adLVgitzsdVxrOr406AppDlUc-HMJ4,2286978
 modules/__init__.py,sha256=G58XleS-EJ2sX4Kehm-3N2m618_W2Es0Kg8CW_eBG7g,327
 modules/ai_actions.py,sha256=i5MJcM-7Y6CAvKUwxmxrVHeoZAVtAP7aRDdWM5KLkO0,33877
 modules/ai_attachment_manager.py,sha256=juZlrW3UPkIkcnj0SREgOQkQROLf0fcu3ShZcKXMxsI,11361
@@ -6,7 +6,7 @@ modules/ai_file_viewer_dialog.py,sha256=lKKqUUlOEVgHmmu6aRxqH7P6ds-7dRLk4ltDyjCw
 modules/autofingers_engine.py,sha256=eJ7tBi7YJvTToe5hYTfnyGXB-qme_cHrOPZibaoR2Xw,17061
 modules/cafetran_docx_handler.py,sha256=_F7Jh0WPVaDnMhdxEsVSXuD1fN9r-S_V6i0gr86Pdfc,14076
 modules/config_manager.py,sha256=MkPY3xVFgFDkcwewLREg4BfyKueO0OJkT1cTLxehcjM,17894
-modules/database_manager.py,sha256=ZdsiuwF67lh-FPKPdalWsW9t6IieX_FM0fA2Bca1xSQ,80221
+modules/database_manager.py,sha256=XbOHaUWq8kdHnT6ZgqPEfA6I3fL5cYiaVXkhYRohi1E,81627
 modules/database_migrations.py,sha256=Y1onFsLDV_6vzJLOpNy3WCZDohBZ2jc4prM-g2_RwLE,14085
 modules/dejavurtf_handler.py,sha256=8NZPPYtHga40SZCypHjPoJPmZTvm9rD-eEUUab7mjtg,28156
 modules/document_analyzer.py,sha256=t1rVvqLaTcpQTEja228C7zZnh8dXshK4wA9t1E9aGVk,19524
@@ -69,7 +69,7 @@ modules/tmx_editor_qt.py,sha256=PxBIUw_06PHYTBHsd8hZzVJXW8T0A0ljfz1Wjjsa4yU,1170
 modules/tmx_generator.py,sha256=pNkxwdMLvSRMMru0lkB1gvViIpg9BQy1EVhRbwoef3k,9426
 modules/tracked_changes.py,sha256=S_BIEC6r7wVAwjG42aSy_RgH4KaMAC8GS5thEvqrYdE,39480
 modules/trados_docx_handler.py,sha256=VPRAQ73cUHs_SEj6x81z1PmSxfjnwPBp9P4fXeK3KpQ,16363
-modules/translation_memory.py,sha256=k0GtO6ANTqxI1XMcv3D5mdAoTgcWlDT5iVsYHizKNUM,28738
+modules/translation_memory.py,sha256=13PDK4_kgYrWTACWBIBypOh2DvoxY9cRT8U6ulilbh4,28739
 modules/translation_results_panel.py,sha256=DmEe0pZRSfcZFg2cWeEREK7H9vrTcPkgeuMW54Pgrys,92505
 modules/translation_services.py,sha256=lyVpWuZK1wtVtYZMDMdLoq1DHBoSaeAnp-Yejb0TlVQ,10530
 modules/unified_prompt_library.py,sha256=lzbevgjUz_qCiYSf141BB0mmuaDhSsevWju_a7welu0,26008
@@ -77,9 +77,9 @@ modules/unified_prompt_manager_qt.py,sha256=fyF3_r0N8hnImT-CcWo1AuBOQ1Dn_ExeeUCk
 modules/voice_commands.py,sha256=iBb-gjWxRMLhFH7-InSRjYJz1EIDBNA2Pog8V7TtJaY,38516
 modules/voice_dictation.py,sha256=QmitXfkG-vRt5hIQATjphHdhXfqmwhzcQcbXB6aRzIg,16386
 modules/voice_dictation_lite.py,sha256=jorY0BmWE-8VczbtGrWwt1zbnOctMoSlWOsQrcufBcc,9423
-supervertaler-1.9.172.dist-info/licenses/LICENSE,sha256=m28u-4qL5nXIWnJ6xlQVw__H30rWFtRK3pCOais2OuY,1092
-supervertaler-1.9.172.dist-info/METADATA,sha256=cDxLhwPym42J-CQfuc07BvtDUoj231w1Lyfz2_iIN6Y,48267
-supervertaler-1.9.172.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
-supervertaler-1.9.172.dist-info/entry_points.txt,sha256=NP4hiCvx-_30YYKqgr-jfJYQvHr1qTYBMfoVmKIXSM8,53
-supervertaler-1.9.172.dist-info/top_level.txt,sha256=9tUHBYUSfaE4S2E4W3eavJsDyYymkwLfeWAHHAPT6Dk,22
-supervertaler-1.9.172.dist-info/RECORD,,
+supervertaler-1.9.173.dist-info/licenses/LICENSE,sha256=m28u-4qL5nXIWnJ6xlQVw__H30rWFtRK3pCOais2OuY,1092
+supervertaler-1.9.173.dist-info/METADATA,sha256=4y6XCqM8xqGT4T9dLHXLn4-KUDkYRAqJ_zsoZUHGW8k,48701
+supervertaler-1.9.173.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
+supervertaler-1.9.173.dist-info/entry_points.txt,sha256=NP4hiCvx-_30YYKqgr-jfJYQvHr1qTYBMfoVmKIXSM8,53
+supervertaler-1.9.173.dist-info/top_level.txt,sha256=9tUHBYUSfaE4S2E4W3eavJsDyYymkwLfeWAHHAPT6Dk,22
+supervertaler-1.9.173.dist-info/RECORD,,