supervertaler 1.9.172__py3-none-any.whl → 1.9.180__py3-none-any.whl

modules/database_manager.py

@@ -17,12 +17,38 @@ import sqlite3
 import os
 import json
 import hashlib
+import unicodedata
+import re
 from datetime import datetime
 from typing import List, Dict, Optional, Tuple
 from pathlib import Path
 from difflib import SequenceMatcher
 
 
+def _normalize_for_matching(text: str) -> str:
+    """Normalize text for exact matching.
+
+    Handles invisible differences that would cause exact match to fail:
+    - Unicode normalization (NFC)
+    - Multiple whitespace -> single space
+    - Leading/trailing whitespace
+    - Non-breaking spaces -> regular spaces
+    """
+    if not text:
+        return ""
+    # Unicode normalize (NFC form)
+    text = unicodedata.normalize('NFC', text)
+    # Convert non-breaking spaces and other whitespace to regular space
+    text = text.replace('\u00a0', ' ')  # NBSP
+    text = text.replace('\u2007', ' ')  # Figure space
+    text = text.replace('\u202f', ' ')  # Narrow NBSP
+    # Collapse multiple whitespace to single space
+    text = re.sub(r'\s+', ' ', text)
+    # Strip leading/trailing whitespace
+    text = text.strip()
+    return text
+
+
 class DatabaseManager:
     """Manages SQLite database for translation resources"""
     
@@ -655,22 +681,46 @@ class DatabaseManager:
     # TRANSLATION MEMORY METHODS
     # ============================================
     
-    def add_translation_unit(self, source: str, target: str, source_lang: str, 
+    def add_translation_unit(self, source: str, target: str, source_lang: str,
                             target_lang: str, tm_id: str = 'project',
                             project_id: str = None, context_before: str = None,
-                            context_after: str = None, notes: str = None) -> int:
+                            context_after: str = None, notes: str = None,
+                            overwrite: bool = False) -> int:
         """
         Add translation unit to database
-        
+
+        Args:
+            source: Source text
+            target: Target text
+            source_lang: Source language code
+            target_lang: Target language code
+            tm_id: TM identifier
+            project_id: Optional project ID
+            context_before: Optional context before
+            context_after: Optional context after
+            notes: Optional notes
+            overwrite: If True, delete existing entries with same source before inserting
+                      (implements "Save only latest translation" mode)
+
         Returns: ID of inserted/updated entry
         """
-        # Generate hash for fast exact matching
-        source_hash = hashlib.md5(source.encode('utf-8')).hexdigest()
-        
+        # Generate hash from NORMALIZED source for consistent exact matching
+        # This handles invisible differences like Unicode normalization, whitespace variations
+        normalized_source = _normalize_for_matching(source)
+        source_hash = hashlib.md5(normalized_source.encode('utf-8')).hexdigest()
+
         try:
+            # If overwrite mode, delete ALL existing entries with same source_hash and tm_id
+            # This ensures only the latest translation is kept
+            if overwrite:
+                self.cursor.execute("""
+                    DELETE FROM translation_units
+                    WHERE source_hash = ? AND tm_id = ?
+                """, (source_hash, tm_id))
+
             self.cursor.execute("""
-                INSERT INTO translation_units 
-                (source_text, target_text, source_lang, target_lang, tm_id, 
+                INSERT INTO translation_units
+                (source_text, target_text, source_lang, target_lang, tm_id,
                  project_id, context_before, context_after, source_hash, notes)
                 VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
                 ON CONFLICT(source_hash, target_text, tm_id) DO UPDATE SET
@@ -678,42 +728,47 @@ class DatabaseManager:
                     modified_date = CURRENT_TIMESTAMP
             """, (source, target, source_lang, target_lang, tm_id,
                   project_id, context_before, context_after, source_hash, notes))
-            
+
             self.connection.commit()
             return self.cursor.lastrowid
-            
+
         except Exception as e:
             self.log(f"Error adding translation unit: {e}")
             return None
     
     def get_exact_match(self, source: str, tm_ids: List[str] = None,
-                       source_lang: str = None, target_lang: str = None, 
+                       source_lang: str = None, target_lang: str = None,
                        bidirectional: bool = True) -> Optional[Dict]:
         """
         Get exact match from TM
-        
+
         Args:
             source: Source text to match
             tm_ids: List of TM IDs to search (None = all)
             source_lang: Filter by source language (base code matching: 'en' matches 'en-US', 'en-GB', etc.)
             target_lang: Filter by target language (base code matching)
             bidirectional: If True, search both directions (nl→en AND en→nl)
-        
+
         Returns: Dictionary with match data or None
         """
         from modules.tmx_generator import get_base_lang_code
-        
+
+        # Try both normalized and non-normalized hashes for backward compatibility
+        # This handles invisible differences like Unicode normalization, whitespace variations
         source_hash = hashlib.md5(source.encode('utf-8')).hexdigest()
-        
+        normalized_source = _normalize_for_matching(source)
+        normalized_hash = hashlib.md5(normalized_source.encode('utf-8')).hexdigest()
+
         # Get base language codes for comparison
         src_base = get_base_lang_code(source_lang) if source_lang else None
         tgt_base = get_base_lang_code(target_lang) if target_lang else None
-        
+
+        # Search using both original hash and normalized hash
         query = """
-            SELECT * FROM translation_units 
-            WHERE source_hash = ? AND source_text = ?
+            SELECT * FROM translation_units
+            WHERE (source_hash = ? OR source_hash = ?)
         """
-        params = [source_hash, source]
+        params = [source_hash, normalized_hash]
         
         if tm_ids:
             placeholders = ','.join('?' * len(tm_ids))
@@ -1422,120 +1477,225 @@ class DatabaseManager:
         # TODO: Implement in Phase 3
         pass
     
-    def search_termbases(self, search_term: str, source_lang: str = None, 
+    def search_termbases(self, search_term: str, source_lang: str = None,
                         target_lang: str = None, project_id: str = None,
-                        min_length: int = 0) -> List[Dict]:
+                        min_length: int = 0, bidirectional: bool = True) -> List[Dict]:
         """
-        Search termbases for matching source terms
-        
+        Search termbases for matching terms (bidirectional by default)
+
         Args:
-            search_term: Source term to search for
+            search_term: Term to search for
             source_lang: Filter by source language (optional)
             target_lang: Filter by target language (optional)
             project_id: Filter by project (optional)
             min_length: Minimum term length to return
-            
+            bidirectional: If True, also search target_term and swap results (default True)
+
         Returns:
             List of termbase hits, sorted by priority (lower = higher priority)
+            Each result includes 'match_direction' ('source' or 'target') indicating
+            which column matched. For 'target' matches, source_term and target_term
+            are swapped so results are always oriented correctly for the current project.
         """
         # Build query with filters - include termbase name and ranking via JOIN
         # Note: termbase_id is stored as TEXT in termbase_terms but INTEGER in termbases
         # Use CAST to ensure proper comparison
         # IMPORTANT: Join with termbase_activation to get the ACTUAL priority for this project
         # CRITICAL FIX: Also match when search_term starts with the glossary term
-        # This handles cases like searching for "ca." when glossary has "ca." 
+        # This handles cases like searching for "ca." when glossary has "ca."
         # AND searching for "ca" when glossary has "ca."
         # We also strip trailing punctuation from glossary terms for comparison
-        query = """
-            SELECT 
-                t.id, t.source_term, t.target_term, t.termbase_id, t.priority, 
+
+        # Build matching conditions for a given column
+        def build_match_conditions(column: str) -> str:
+            return f"""(
+                LOWER(t.{column}) = LOWER(?) OR
+                LOWER(t.{column}) LIKE LOWER(?) OR
+                LOWER(t.{column}) LIKE LOWER(?) OR
+                LOWER(t.{column}) LIKE LOWER(?) OR
+                LOWER(RTRIM(t.{column}, '.!?,;:')) = LOWER(?) OR
+                LOWER(?) LIKE LOWER(t.{column}) || '%' OR
+                LOWER(?) = LOWER(RTRIM(t.{column}, '.!?,;:'))
+            )"""
+
+        # Build match params for one direction
+        def build_match_params() -> list:
+            return [
+                search_term,
+                f"{search_term} %",
+                f"% {search_term}",
+                f"% {search_term} %",
+                search_term,  # For RTRIM comparison
+                search_term,  # For reverse LIKE
+                search_term   # For reverse RTRIM comparison
+            ]
+
+        # Matching patterns:
+        # 1. Exact match: column = search_term
+        # 2. Glossary term starts with search: column LIKE "search_term %"
+        # 3. Glossary term ends with search: column LIKE "% search_term"
+        # 4. Glossary term contains search: column LIKE "% search_term %"
+        # 5. Glossary term (stripped) = search_term: RTRIM(column) = search_term (handles "ca." = "ca")
+        # 6. Search starts with glossary term: search_term LIKE column || '%'
+        # 7. Search = glossary term stripped: search_term = RTRIM(column)
+
+        # Base SELECT for forward matches (source_term matches)
+        base_select_forward = """
+            SELECT
+                t.id, t.source_term, t.target_term, t.termbase_id, t.priority,
                 t.forbidden, t.source_lang, t.target_lang, t.definition, t.domain,
                 t.notes, t.project, t.client,
                 tb.name as termbase_name,
                 tb.source_lang as termbase_source_lang,
                 tb.target_lang as termbase_target_lang,
                 tb.is_project_termbase,
-                COALESCE(ta.priority, tb.ranking) as ranking
+                COALESCE(ta.priority, tb.ranking) as ranking,
+                'source' as match_direction
             FROM termbase_terms t
             LEFT JOIN termbases tb ON CAST(t.termbase_id AS INTEGER) = tb.id
             LEFT JOIN termbase_activation ta ON ta.termbase_id = tb.id AND ta.project_id = ? AND ta.is_active = 1
-            WHERE (
-                LOWER(t.source_term) = LOWER(?) OR 
-                LOWER(t.source_term) LIKE LOWER(?) OR 
-                LOWER(t.source_term) LIKE LOWER(?) OR 
-                LOWER(t.source_term) LIKE LOWER(?) OR
-                LOWER(RTRIM(t.source_term, '.!?,;:')) = LOWER(?) OR
-                LOWER(?) LIKE LOWER(t.source_term) || '%' OR
-                LOWER(?) = LOWER(RTRIM(t.source_term, '.!?,;:'))
-            )
+            WHERE {match_conditions}
             AND (ta.is_active = 1 OR tb.is_project_termbase = 1)
-        """
-        # Matching patterns:
-        # 1. Exact match: source_term = search_term
-        # 2. Glossary term starts with search: source_term LIKE "search_term %"
-        # 3. Glossary term ends with search: source_term LIKE "% search_term"
-        # 4. Glossary term contains search: source_term LIKE "% search_term %"
-        # 5. Glossary term (stripped) = search_term: RTRIM(source_term) = search_term (handles "ca." = "ca")
-        # 6. Search starts with glossary term: search_term LIKE source_term || '%' 
-        # 7. Search = glossary term stripped: search_term = RTRIM(source_term)
-        params = [
-            project_id if project_id else 0,  # Use 0 if no project (won't match any activation records)
-            search_term,
-            f"{search_term} %",
-            f"% {search_term}",
-            f"% {search_term} %",
-            search_term,  # For RTRIM comparison
-            search_term,  # For reverse LIKE
-            search_term   # For reverse RTRIM comparison
-        ]
-        
-        # Language filters - if term has no language, use termbase language for filtering
+        """.format(match_conditions=build_match_conditions('source_term'))
+
+        # Base SELECT for reverse matches (target_term matches) - swap source/target in output
+        base_select_reverse = """
+            SELECT
+                t.id, t.target_term as source_term, t.source_term as target_term,
+                t.termbase_id, t.priority,
+                t.forbidden, t.target_lang as source_lang, t.source_lang as target_lang,
+                t.definition, t.domain,
+                t.notes, t.project, t.client,
+                tb.name as termbase_name,
+                tb.target_lang as termbase_source_lang,
+                tb.source_lang as termbase_target_lang,
+                tb.is_project_termbase,
+                COALESCE(ta.priority, tb.ranking) as ranking,
+                'target' as match_direction
+            FROM termbase_terms t
+            LEFT JOIN termbases tb ON CAST(t.termbase_id AS INTEGER) = tb.id
+            LEFT JOIN termbase_activation ta ON ta.termbase_id = tb.id AND ta.project_id = ? AND ta.is_active = 1
+            WHERE {match_conditions}
+            AND (ta.is_active = 1 OR tb.is_project_termbase = 1)
+        """.format(match_conditions=build_match_conditions('target_term'))
+
+        # Build params
+        project_param = project_id if project_id else 0
+        forward_params = [project_param] + build_match_params()
+        reverse_params = [project_param] + build_match_params()
+
+        # Build language filter conditions
+        lang_conditions_forward = ""
+        lang_conditions_reverse = ""
+        lang_params_forward = []
+        lang_params_reverse = []
+
         if source_lang:
-            query += """ AND (
-                t.source_lang = ? OR 
+            # For forward: filter on source_lang
+            lang_conditions_forward += """ AND (
+                t.source_lang = ? OR
                 (t.source_lang IS NULL AND tb.source_lang = ?) OR
                 (t.source_lang IS NULL AND tb.source_lang IS NULL)
             )"""
-            params.extend([source_lang, source_lang])
-        
+            lang_params_forward.extend([source_lang, source_lang])
+            # For reverse: source_lang becomes target_lang (swapped)
+            lang_conditions_reverse += """ AND (
+                t.target_lang = ? OR
+                (t.target_lang IS NULL AND tb.target_lang = ?) OR
+                (t.target_lang IS NULL AND tb.target_lang IS NULL)
+            )"""
+            lang_params_reverse.extend([source_lang, source_lang])
+
         if target_lang:
-            query += """ AND (
-                t.target_lang = ? OR 
+            # For forward: filter on target_lang
+            lang_conditions_forward += """ AND (
+                t.target_lang = ? OR
                 (t.target_lang IS NULL AND tb.target_lang = ?) OR
                 (t.target_lang IS NULL AND tb.target_lang IS NULL)
             )"""
-            params.extend([target_lang, target_lang])
-        
-        # Project filter: match project-specific terms OR global terms (project_id IS NULL)
+            lang_params_forward.extend([target_lang, target_lang])
+            # For reverse: target_lang becomes source_lang (swapped)
+            lang_conditions_reverse += """ AND (
+                t.source_lang = ? OR
+                (t.source_lang IS NULL AND tb.source_lang = ?) OR
+                (t.source_lang IS NULL AND tb.source_lang IS NULL)
+            )"""
+            lang_params_reverse.extend([target_lang, target_lang])
+
+        # Project filter conditions
+        project_conditions = ""
+        project_params = []
         if project_id:
-            query += " AND (t.project_id = ? OR t.project_id IS NULL)"
-            params.append(project_id)
-        
+            project_conditions = " AND (t.project_id = ? OR t.project_id IS NULL)"
+            project_params = [project_id]
+
+        # Min length conditions
+        min_len_forward = ""
+        min_len_reverse = ""
         if min_length > 0:
-            query += f" AND LENGTH(t.source_term) >= {min_length}"
-        
-        # Sort by ranking (lower number = higher priority)
-        # Project termbases (ranking IS NULL) appear first, then by ranking, then alphabetically
-        # Use COALESCE to treat NULL as -1 (highest priority)
-        query += " ORDER BY COALESCE(tb.ranking, -1) ASC, t.source_term ASC"
-        
+            min_len_forward = f" AND LENGTH(t.source_term) >= {min_length}"
+            min_len_reverse = f" AND LENGTH(t.target_term) >= {min_length}"
+
+        # Build forward query
+        forward_query = base_select_forward + lang_conditions_forward + project_conditions + min_len_forward
+        forward_params.extend(lang_params_forward)
+        forward_params.extend(project_params)
+
+        if bidirectional:
+            # Build reverse query
+            reverse_query = base_select_reverse + lang_conditions_reverse + project_conditions + min_len_reverse
+            reverse_params.extend(lang_params_reverse)
+            reverse_params.extend(project_params)
+
+            # Combine with UNION and sort
+            query = f"""
+                SELECT * FROM (
+                    {forward_query}
+                    UNION ALL
+                    {reverse_query}
+                ) combined
+                ORDER BY COALESCE(ranking, -1) ASC, source_term ASC
+            """
+            params = forward_params + reverse_params
+        else:
+            # Original forward-only behavior
+            query = forward_query + " ORDER BY COALESCE(ranking, -1) ASC, source_term ASC"
+            params = forward_params
+
         self.cursor.execute(query, params)
         results = []
+        seen_combinations = set()  # Track (source_term, target_term, termbase_id) to avoid duplicates
+
         for row in self.cursor.fetchall():
             result_dict = dict(row)
+
+            # Deduplicate: same term pair from same termbase should only appear once
+            # Prefer 'source' match over 'target' match
+            combo_key = (
+                result_dict.get('source_term', '').lower(),
+                result_dict.get('target_term', '').lower(),
+                result_dict.get('termbase_id')
+            )
+            if combo_key in seen_combinations:
+                continue
+            seen_combinations.add(combo_key)
+
             # SQLite stores booleans as 0/1, explicitly convert to Python bool
             if 'is_project_termbase' in result_dict:
                 result_dict['is_project_termbase'] = bool(result_dict['is_project_termbase'])
-            
+
             # Fetch target synonyms for this term and include them in the result
             term_id = result_dict.get('id')
+            match_direction = result_dict.get('match_direction', 'source')
             if term_id:
                 try:
+                    # For reverse matches, fetch 'source' synonyms since they become targets
+                    synonym_lang = 'source' if match_direction == 'target' else 'target'
                     self.cursor.execute("""
                         SELECT synonym_text, forbidden FROM termbase_synonyms
-                        WHERE term_id = ? AND language = 'target'
+                        WHERE term_id = ? AND language = ?
                         ORDER BY display_order ASC
-                    """, (term_id,))
+                    """, (term_id, synonym_lang))
                     synonyms = []
                     for syn_row in self.cursor.fetchall():
                         syn_text = syn_row[0]
@@ -1545,7 +1705,7 @@ class DatabaseManager:
                     result_dict['target_synonyms'] = synonyms
                 except Exception:
                     result_dict['target_synonyms'] = []
-            
+
             results.append(result_dict)
         return results
     

modules/database_migrations.py

@@ -186,9 +186,13 @@ def run_all_migrations(db_manager) -> bool:
     # Migration 3: Add display_order and forbidden fields to synonyms
     if not migrate_synonym_fields(db_manager):
         success = False
-    
+
+    # Migration 4: Add ai_inject field to termbases
+    if not migrate_termbase_ai_inject(db_manager):
+        success = False
+
     print("="*60)
-    
+
     return success
 
 
@@ -221,18 +225,26 @@ def check_and_migrate(db_manager) -> bool:
         
         # Check if synonyms table exists
         cursor.execute("""
-            SELECT name FROM sqlite_master 
+            SELECT name FROM sqlite_master
             WHERE type='table' AND name='termbase_synonyms'
         """)
         needs_synonyms_table = cursor.fetchone() is None
-        
+
+        # Check if termbases table has ai_inject column
+        cursor.execute("PRAGMA table_info(termbases)")
+        termbase_columns = {row[1] for row in cursor.fetchall()}
+        needs_ai_inject = 'ai_inject' not in termbase_columns
+
         if needs_migration:
             print(f"⚠️ Migration needed - missing columns: {', '.join([c for c in ['project', 'client', 'term_uuid', 'note'] if c not in columns])}")
-        
+
         if needs_synonyms_table:
             print("⚠️ Migration needed - termbase_synonyms table missing")
-        
-        if needs_migration or needs_synonyms_table:
+
+        if needs_ai_inject:
+            print("⚠️ Migration needed - termbases.ai_inject column missing")
+
+        if needs_migration or needs_synonyms_table or needs_ai_inject:
             success = run_all_migrations(db_manager)
             if success:
                 # Generate UUIDs for terms that don't have them
@@ -316,6 +328,41 @@ def migrate_synonym_fields(db_manager) -> bool:
         return False
 
 
+def migrate_termbase_ai_inject(db_manager) -> bool:
+    """
+    Add ai_inject column to termbases table.
+    When enabled, the termbase's terms will be injected into LLM translation prompts.
+
+    Args:
+        db_manager: DatabaseManager instance
+
+    Returns:
+        True if migration successful
+    """
+    try:
+        cursor = db_manager.cursor
+
+        # Check which columns exist
+        cursor.execute("PRAGMA table_info(termbases)")
+        columns = {row[1] for row in cursor.fetchall()}
+
+        if 'ai_inject' not in columns:
+            print("📊 Adding 'ai_inject' column to termbases...")
+            cursor.execute("ALTER TABLE termbases ADD COLUMN ai_inject BOOLEAN DEFAULT 0")
+            db_manager.connection.commit()
+            print("  ✓ Column 'ai_inject' added successfully")
+        else:
+            print("✅ termbases.ai_inject column already exists")
+
+        return True
+
+    except Exception as e:
+        print(f"❌ ai_inject migration failed: {e}")
+        import traceback
+        traceback.print_exc()
+        return False
+
+
 def generate_missing_uuids(db_manager) -> bool:
     """
     Generate UUIDs for any termbase terms that don't have them.

modules/mqxliff_handler.py

@@ -159,9 +159,78 @@ class MQXLIFFHandler:
                 
                 segment = FormattedSegment(trans_unit_id, plain_text, formatted_xml)
                 segments.append(segment)
-        
+
         return segments
-    
+
+    def extract_bilingual_segments(self) -> List[Dict]:
+        """
+        Extract all source AND target segments from the MQXLIFF file.
+        Used for importing pretranslated mqxliff files.
+
+        Returns:
+            List of dicts with 'id', 'source', 'target', 'status' keys
+        """
+        segments = []
+
+        if self.body_element is None:
+            return segments
+
+        # Find all trans-unit elements (with or without namespace)
+        trans_units = self.body_element.findall('.//xliff:trans-unit', self.NAMESPACES)
+        if not trans_units:
+            trans_units = self.body_element.findall('.//trans-unit')
+
+        for trans_unit in trans_units:
+            trans_unit_id = trans_unit.get('id', 'unknown')
+
+            # Skip auxiliary segments (like hyperlink URLs with mq:nosplitjoin="true")
+            nosplitjoin = trans_unit.get('{MQXliff}nosplitjoin', 'false')
+            if nosplitjoin == 'true':
+                continue
+
+            # Find source element
+            source_elem = trans_unit.find('xliff:source', self.NAMESPACES)
+            if source_elem is None:
+                source_elem = trans_unit.find('source')
+
+            # Find target element
+            target_elem = trans_unit.find('xliff:target', self.NAMESPACES)
+            if target_elem is None:
+                target_elem = trans_unit.find('target')
+
+            source_text = ""
+            target_text = ""
+
+            if source_elem is not None:
+                source_text = self._extract_plain_text(source_elem)
+
+            if target_elem is not None:
+                target_text = self._extract_plain_text(target_elem)
+
+            # Get memoQ status if available
+            mq_status = trans_unit.get('{MQXliff}status', '')
+
+            # Map memoQ status to internal status
+            # memoQ statuses: "NotStarted", "Editing", "Confirmed", "Reviewed", "Rejected", etc.
+            status = 'not_started'
+            if mq_status in ['Confirmed', 'ProofRead', 'Reviewed']:
+                status = 'confirmed'
+            elif mq_status == 'Editing':
+                status = 'translated'
+            elif target_text.strip():
+                # Has target but unknown status - mark as pre-translated
+                status = 'pre_translated'
+
+            segments.append({
+                'id': trans_unit_id,
+                'source': source_text,
+                'target': target_text,
+                'status': status,
+                'mq_status': mq_status
+            })
+
+        return segments
+
     def _extract_plain_text(self, element: ET.Element) -> str:
         """
         Recursively extract plain text from an XML element, stripping all tags.