debase 0.6.2__py3-none-any.whl → 0.7.0__py3-none-any.whl

debase/lineage_format.py

@@ -213,8 +213,8 @@ class VariantRecord:
         return result
 
 
-    def ttn_or_yield(self) -> Optional[float]:
-        for col in ("ttn", "yield"):
+    def get_fitness_value(self) -> Optional[float]:
+        for col in ("ttn", "tof", "yield"):
             val = self.row.get(col)
             if val is not None and pd.notna(val):
                 try:
@@ -1272,9 +1272,15 @@ def flatten_dataframe(df: pd.DataFrame) -> pd.DataFrame:
 
         # Fitness type -------------------------------------------------------
         fitness_type = ""
-        if rec.ttn_or_yield() is not None:
+        if rec.get_fitness_value() is not None:
             ttn_val = row.get("ttn")
-            fitness_type = "ttn" if (ttn_val is not None and pd.notna(ttn_val)) else "yield"
+            tof_val = row.get("tof")
+            if ttn_val is not None and pd.notna(ttn_val):
+                fitness_type = "ttn"
+            elif tof_val is not None and pd.notna(tof_val):
+                fitness_type = "tof"
+            else:
+                fitness_type = "yield"
         
         # Additional info -----------------------------------------------------
         extra: Dict[str, str] = {
@@ -1295,7 +1301,7 @@ def flatten_dataframe(df: pd.DataFrame) -> pd.DataFrame:
             amino_acid_substitutions=aa_muts,
             nt_sequence=rec.nt_seq,
             aa_sequence=rec.aa_seq,
-            fitness_value=rec.ttn_or_yield(),
+            fitness_value=rec.get_fitness_value(),
             fitness_type=fitness_type,
             cofactor=cofactor,
             reaction_condition=reaction_condition,

debase/reaction_info_extractor.py

@@ -29,6 +29,7 @@ import json
 import logging
 import os
 import re
+import subprocess
 import sys
 import time
 from base64 import b64encode, b64decode
@@ -90,6 +91,40 @@ handler.setFormatter(logging.Formatter("%(levelname)s [%(name)s] %(message)s"))
 LOGGER.addHandler(handler)
 LOGGER.setLevel(logging.INFO)
 
+# === OPSIN VALIDATION === -------------------------------------------------
+
+def is_valid_iupac_name_with_opsin(name: str) -> bool:
+    """Check if a name is a valid IUPAC name using the local OPSIN command."""
+    if not name or len(name.strip()) < 3:
+        return False
+    
+    # Skip if it looks like a compound ID (e.g., "1a", "S1", etc.)
+    if re.match(r'^[0-9]+[a-z]?$|^S\d+$', name.strip()):
+        return False
+    
+    try:
+        # Use local OPSIN command to check if name can be converted to SMILES
+        process = subprocess.run(
+            ['opsin', '-o', 'smi'],
+            input=name.strip(),
+            text=True,
+            capture_output=True,
+            timeout=30
+        )
+        
+        # If OPSIN successfully converts to SMILES, the name is valid IUPAC
+        if process.returncode == 0 and process.stdout.strip():
+            output = process.stdout.strip()
+            # Check if output looks like a valid SMILES (contains common SMILES characters)
+            if any(char in output for char in 'CNOS()=[]#+-'):
+                return True
+        
+        return False
+            
+    except Exception as e:
+        LOGGER.debug(f"OPSIN check failed for '{name}': {e}")
+        return False
+
 # --- Debug dump helper ----------------------------------------------------
 def _dump(text: str | bytes, path: Path | str) -> None:
     """Write `text` / `bytes` to `path`, creating parent dirs as needed."""
@@ -606,23 +641,29 @@ Given the following text sections, identify where the MODEL REACTION information
 The model reaction is the STANDARD reaction used to evaluate all enzyme variants 
 (not the substrate scope). Look for:
 
-- Sections titled "Model Reaction", "Standard Reaction", "General Procedure"
-- Text describing the reaction conditions used for enzyme evolution/screening
-- Sections describing which substrates were used as the benchmark
-- Compound numbers (e.g., "6a", "7a") used in the model reaction
+- SPECIFIC compound numbers (e.g., "1a", "2a", "3a") used in the model reaction
+- Reaction SCHEMES or FIGURES showing the model reaction with numbered compounds
+- Tables showing reaction conditions with specific compound IDs
+- Sections titled "Model Reaction", "Standard Reaction", "General Procedure" WITH compound numbers
+
+CRITICAL REQUIREMENTS:
+1. The location MUST reference SPECIFIC numbered compounds (not generic descriptions)
+2. DO NOT use generic locations like "main text" or "introduction"
+3. MUST be a Figure, Scheme, Table, or specific SI section
+4. Look for actual compound IDs like "1a + 2a → 3a" or "substrate 1a"
 
 Also identify where the IUPAC names for these specific compounds are listed.
 
 Respond with a JSON object containing:
 {
   "model_reaction_location": {
-    "location": "section name or description",
+    "location": "SPECIFIC Figure/Scheme/Table number (e.g., 'Figure 2a', 'Scheme 1', 'Table S1')",
     "confidence": 0-100,
-    "reason": "why this contains the model reaction",
-    "compound_ids": ["list", "of", "compound", "IDs", "if", "found"]
+    "reason": "why this contains the model reaction WITH specific compound IDs",
+    "compound_ids": ["list", "of", "SPECIFIC", "compound", "IDs", "found", "e.g.", "1a", "2a", "3a"]
   },
   "conditions_location": {
-    "location": "where reaction conditions are described",
+    "location": "SPECIFIC location where reaction conditions are described",
     "confidence": 0-100
   },
   "iupac_location": {
@@ -632,6 +673,11 @@ Respond with a JSON object containing:
   }
 }
 
+IMPORTANT: 
+- If no SPECIFIC compound IDs are found, set compound_ids to []
+- The model_reaction_location MUST be a Figure, Scheme, Table, or SI section, NOT "main text"
+- Look for numbered compounds like "1a", "2a", not generic terms like "enol acetates"
+
 Respond ONLY with **minified JSON**. NO markdown fences, no commentary.
 """)
 
@@ -642,11 +688,20 @@ This is the reaction used for directed evolution screening, NOT the substrate sc
 Look for terms like "model reaction", "standard substrate", "benchmark reaction", 
 or the specific reaction mentioned in enzyme screening/evolution sections.
 
+CRITICAL STEPS FOR COMPOUND IDENTIFICATION:
+1. ALWAYS look for specific compound IDs/numbers in the model reaction (e.g., "1a", "2a", "3a", "6a", "7a")
+2. If the text mentions generic terms like "enol acetates" or "silyl enol ethers", search for the SPECIFIC numbered compounds used
+3. Look in reaction schemes, figures, and experimental sections for numbered compounds
+4. Common patterns:
+   - "compound 1a" or "substrate 1a"
+   - Numbers in bold or italics (1a, 2a, etc.)
+   - References like "using 1a as substrate"
+
 CRITICAL STEPS FOR IUPAC NAMES:
-1. First identify the compound IDs used in the model reaction (e.g., "6a", "7a")
-2. Then search the provided context for these compound IDs to find their IUPAC names
-3. Look for sections with "Compound 6a", "Product 7a", or similar patterns
-4. The IUPAC names are usually given after the compound ID in parentheses or after a colon
+1. After finding compound IDs, search the context for these IDs to find their IUPAC names
+2. Look for sections with "Compound 1a:", "Product 3a:", or similar patterns
+3. The IUPAC names are usually given after the compound ID in parentheses or after a colon
+4. If no IUPAC name is found for a compound ID, still include the ID in substrate_list/product_list
 
 CRITICAL FOR SUBSTRATE CONCENTRATION:
 - Look carefully in FIGURES and figure captions for substrate concentration information
@@ -657,10 +712,10 @@ CRITICAL FOR SUBSTRATE CONCENTRATION:
 - The substrate is the molecule being chemically transformed by the enzyme
 
 Return a JSON object with:
-  * "substrate_list" - Array of substrate identifiers as used in the paper (e.g., ["5", "6a"])
-  * "substrate_iupac_list" - Array of IUPAC names for ALL substrates/reagents
-  * "product_list" - Array of product identifiers as used in the paper (e.g., ["7a"])
-  * "product_iupac_list" - Array of IUPAC names for ALL products formed
+  * "substrate_list" - Array of substrate identifiers as used in the paper (e.g., ["1a", "2a"]) - NEVER generic descriptions
+  * "substrate_iupac_list" - Array of IUPAC names for ALL substrates/reagents (null if not found)
+  * "product_list" - Array of product identifiers as used in the paper (e.g., ["3a"]) - NEVER generic descriptions
+  * "product_iupac_list" - Array of IUPAC names for ALL products formed (null if not found)
   * "reaction_substrate_concentration" - Concentration of actual substrate(s) being transformed, NOT reducing agents like dithionite
   * "cofactor" - Any cofactors used (e.g., "NADH", "NADPH", "FAD", "heme", etc.) or null if none
   * "reaction_temperature" - reaction temperature (e.g., "25°C", "room temperature")
@@ -669,7 +724,8 @@ Return a JSON object with:
   * "reaction_other_conditions" - other important conditions (enzyme loading, reducing agents like dithionite, time, anaerobic, etc.)
 
 IMPORTANT: 
-- Extract the reaction used for ENZYME EVOLUTION/SCREENING (not substrate scope)
+- ALWAYS use specific compound IDs (like "1a", "2a") in substrate_list and product_list, NEVER generic descriptions
+- If you can't find specific compound IDs, look harder in figures, schemes, and experimental sections
 - Substrate concentration = concentration of chemicals being transformed, NOT reducing agents (dithionite, NADH, etc.)
 - Maintain correspondence: substrate_list[i] should map to substrate_iupac_list[i], same for products
 - If a compound ID has no IUPAC name found, still include it in the list with null in the IUPAC list
@@ -783,7 +839,7 @@ Return as JSON:
 ###############################################################################
 
 class ReactionExtractor:
-    _FIG_RE = re.compile(r"(?:supplementary\s+)?fig(?:ure)?\s+s?\d+[a-z]?", re.I)
+    _FIG_RE = re.compile(r"(?:supplementary\s+)?fig(?:ure)?\.?\s+s?\d+[a-z]?", re.I)
     _TAB_RE = re.compile(r"(?:supplementary\s+)?tab(?:le)?\s+s?\d+[a-z]?", re.I)
 
     def __init__(self, manuscript: Path, si: Optional[Path], cfg: Config, debug_dir: Optional[Path] = None, 
@@ -1408,6 +1464,10 @@ class ReactionExtractor:
                         text = ' '.join(text.split())
                         # Normalize different dash types
                         text = text.replace('–', '-').replace('—', '-')
+                        # Normalize pipe character and other special chars
+                        text = text.replace('|', ' ').replace('│', ' ')
+                        # Remove multiple spaces
+                        text = ' '.join(text.split())
                         return text
                     
                     normalized_hint = normalize_for_matching(caption_hint[:100])  # Use first 100 chars
@@ -1904,15 +1964,17 @@ class ReactionExtractor:
         ref_lc = location_str.lower()
         image_b64: Optional[str] = None
         
-        # First, validate that the location actually exists in the document
-        if not self._validate_location_exists(location_str):
+        # Skip validation entirely when we have a caption hint - trust the vision model
+        if caption_hint:
+            LOGGER.info("Skipping validation - using caption hint for %s", location_str)
+        elif not self._validate_location_exists(location_str):
             LOGGER.warning("Location %s not found in document - skipping", location_str)
             return []
         
         # Add campaign context if available
         campaign_context = ""
         if self.campaign_filter:
-            campaign_context = f"\n\nIMPORTANT: You are extracting data for the {self.campaign_filter} campaign.\nOnly extract data that is relevant to this specific campaign.\n"
+            campaign_context = f"\n\nIMPORTANT: You are extracting data for the {self.campaign_filter} campaign.\nOnly extract data that is relevant to this specific campaign.\nEXCLUDE reference variants from other publications - only include variants created/tested in THIS study.\n"
         
         if self._TAB_RE.search(ref_lc):
             # For tables, try to extract the page as an image first
@@ -1976,6 +2038,24 @@ class ReactionExtractor:
             prompt = campaign_context + location_context + PROMPT_EXTRACT_FIGURE_METRICS_BATCH.format(enzyme_names=enzyme_names)
             LOGGER.info("Gemini Vision: extracting metrics for %d enzymes from %s…", len(enzyme_list), ref)
             tag = f"extract_metrics_batch_vision"
+            
+            # Save the figure image to debug directory
+            if self.debug_dir and isinstance(ref, dict):
+                location_str = ref.get('location', str(ref))
+            else:
+                location_str = str(ref)
+            
+            if self.debug_dir:
+                timestamp = int(time.time())
+                img_file = self.debug_dir / f"metrics_extraction_{location_str.replace(' ', '_').replace('.', '')}_{timestamp}.png"
+                try:
+                    import base64
+                    img_bytes = base64.b64decode(image_b64)
+                    with open(img_file, 'wb') as f:
+                        f.write(img_bytes)
+                    LOGGER.info("Saved metrics extraction figure to: %s", img_file)
+                except Exception as e:
+                    LOGGER.warning("Failed to save metrics extraction figure: %s", e)
         else:
             # Add enzyme names to prompt for batch extraction with explicit format requirement
             format_example = '{"enzyme1": {"yield": "99.0%", "ttn": null, ...}, "enzyme2": {"yield": "85.0%", ...}}'
@@ -2112,6 +2192,10 @@ These variants belong to campaign: {self.campaign_filter}
 {campaigns_context}
 Focus on finding the model reaction that was used to evaluate THESE specific variants.
 Different campaigns may use different model reactions.
+
+CRITICAL: These variants should be from THIS study only!
+- EXCLUDE any reference variants cited from other publications
+- Only include variants that were created/engineered in this manuscript
 """
         
         prompt = enzyme_context + PROMPT_FIND_MODEL_REACTION_LOCATION + "\n\n=== CAPTIONS AND SECTIONS ===\n" + all_text + "\n\n=== MANUSCRIPT TEXT PREVIEW ===\n" + ms_preview + "\n\n=== SI TEXT PREVIEW ===\n" + si_preview
@@ -2843,6 +2927,12 @@ These variants belong to campaign: {self.campaign_filter}
 Focus on extracting the model reaction that was used to evaluate THESE specific variants.
 Different campaigns may use different model reactions and substrates.
 
+CRITICAL: EXCLUDE reference variants from other publications!
+- Only extract data for variants that were actually tested/created in THIS study
+- Do NOT include data for reference enzymes cited from other papers
+- Look for phrases like "from reference", "previously reported", "from [Author] et al." to identify reference variants
+- Focus ONLY on the variants that were engineered/tested in this manuscript
+
 """
         
         # Include both manuscript and SI text for better coverage
@@ -3042,6 +3132,100 @@ Different campaigns may use different model reactions and substrates.
         ]
         for key in expected_keys:
             data.setdefault(key, None)
+        
+        # === OPSIN VALIDATION AND COMPOUND MAPPING FALLBACK ===
+        # Check if the IUPAC names are actually valid using OPSIN
+        needs_compound_mapping = False
+        
+        # Check substrate IUPAC names
+        substrate_has_invalid = False
+        if data.get("substrate_list") and isinstance(data["substrate_list"], list):
+            # Check if we have substrate IDs but missing or invalid IUPAC names
+            if not data.get("substrate_iupac_list"):
+                LOGGER.warning("Substrate list exists but no IUPAC names provided")
+                substrate_has_invalid = True
+            else:
+                substrate_names = data["substrate_iupac_list"].split("; ") if isinstance(data["substrate_iupac_list"], str) else []
+                # Check each substrate ID has a valid IUPAC name
+                for i, substrate_id in enumerate(data["substrate_list"]):
+                    if i >= len(substrate_names) or not substrate_names[i]:
+                        LOGGER.warning(f"No IUPAC name for substrate '{substrate_id}'")
+                        substrate_has_invalid = True
+                    elif not is_valid_iupac_name_with_opsin(substrate_names[i]):
+                        LOGGER.warning(f"Invalid IUPAC name detected for substrate '{substrate_id}': '{substrate_names[i]}'")
+                        substrate_has_invalid = True
+            
+            if substrate_has_invalid:
+                needs_compound_mapping = True
+                LOGGER.info("Found missing or invalid substrate IUPAC names, will attempt compound mapping")
+        
+        # Check product IUPAC names
+        product_has_invalid = False
+        if data.get("product_list") and isinstance(data["product_list"], list):
+            # Check if we have product IDs but missing or invalid IUPAC names
+            if not data.get("product_iupac_list"):
+                LOGGER.warning("Product list exists but no IUPAC names provided")
+                product_has_invalid = True
+            else:
+                product_names = data["product_iupac_list"].split("; ") if isinstance(data["product_iupac_list"], str) else []
+                # Check each product ID has a valid IUPAC name
+                for i, product_id in enumerate(data["product_list"]):
+                    if i >= len(product_names) or not product_names[i]:
+                        LOGGER.warning(f"No IUPAC name for product '{product_id}'")
+                        product_has_invalid = True
+                    elif not is_valid_iupac_name_with_opsin(product_names[i]):
+                        LOGGER.warning(f"Invalid IUPAC name detected for product '{product_id}': '{product_names[i]}'")
+                        product_has_invalid = True
+            
+            if product_has_invalid:
+                needs_compound_mapping = True
+                LOGGER.info("Found missing or invalid product IUPAC names, will attempt compound mapping")
+        
+        # If we need compound mapping and have substrate/product lists, attempt it
+        if needs_compound_mapping and (data.get("substrate_list") or data.get("product_list")):
+            LOGGER.info("Attempting compound mapping due to invalid IUPAC names")
+            
+            # Collect all compound IDs that need mapping
+            compound_ids_to_map = []
+            if data.get("substrate_list") and isinstance(data["substrate_list"], list):
+                compound_ids_to_map.extend(data["substrate_list"])
+            if data.get("product_list") and isinstance(data["product_list"], list):
+                compound_ids_to_map.extend(data["product_list"])
+            
+            if compound_ids_to_map:
+                LOGGER.info(f"Attempting to map compound IDs: {compound_ids_to_map}")
+                
+                # Use the adaptive compound mapping
+                compound_mappings = self._extract_compound_mappings_adaptive(
+                    compound_ids_to_map,
+                    campaign_filter=self.campaign_filter
+                )
+                
+                # Re-map substrate IUPAC names
+                if data.get("substrate_list") and isinstance(data["substrate_list"], list):
+                    mapped_substrates = []
+                    for substrate_id in data["substrate_list"]:
+                        mapping = compound_mappings.get(substrate_id.lower().strip())
+                        if mapping and mapping.iupac_name and is_valid_iupac_name_with_opsin(mapping.iupac_name):
+                            mapped_substrates.append(mapping.iupac_name)
+                            LOGGER.info(f"Successfully mapped substrate '{substrate_id}' to IUPAC: {mapping.iupac_name}")
+                    
+                    if mapped_substrates:
+                        data["substrate_iupac_list"] = "; ".join(mapped_substrates)
+                        LOGGER.info(f"Updated substrate IUPAC list with {len(mapped_substrates)} valid names")
+                
+                # Re-map product IUPAC names
+                if data.get("product_list") and isinstance(data["product_list"], list):
+                    mapped_products = []
+                    for product_id in data["product_list"]:
+                        mapping = compound_mappings.get(product_id.lower().strip())
+                        if mapping and mapping.iupac_name and is_valid_iupac_name_with_opsin(mapping.iupac_name):
+                            mapped_products.append(mapping.iupac_name)
+                            LOGGER.info(f"Successfully mapped product '{product_id}' to IUPAC: {mapping.iupac_name}")
+                    
+                    if mapped_products:
+                        data["product_iupac_list"] = "; ".join(mapped_products)
+                        LOGGER.info(f"Updated product IUPAC list with {len(mapped_products)} valid names")
             
         return data
 
@@ -3120,21 +3304,10 @@ Different campaigns may use different model reactions and substrates.
             # Extract model reaction for this location - use unified approach
             LOGGER.info("Extracting model reaction for location: %s", best_location['location'])
             
-            # Try lineage-specific extraction first
-            location_model_reaction = self.find_lineage_model_reaction(
-                best_location['location'], 
-                location_context,
-                model_reaction_locations
-            )
-            
-            # Check if lineage extraction was successful
-            if location_model_reaction.get('substrate_ids') or location_model_reaction.get('product_ids'):
-                LOGGER.info("Using lineage-specific model reaction data")
-                model_info = self._extract_lineage_model_info(location_model_reaction, location_enzymes)
-            else:
-                LOGGER.info("Lineage extraction failed, using comprehensive multimodal extraction")
-                # Use the comprehensive multimodal approach as fallback
-                model_info = self.gather_model_reaction_info(location_enzymes)
+            # Skip lineage-specific extraction and use comprehensive multimodal extraction directly
+            # The lineage-specific extraction often returns generic substrate classes instead of specific compounds
+            LOGGER.info("Using comprehensive multimodal extraction for model reaction")
+            model_info = self.gather_model_reaction_info(location_enzymes)
                 
             LOGGER.info("Model reaction extraction complete for location: %s", best_location['location'])