isa-model 0.4.0__py3-none-any.whl → 0.4.4__py3-none-any.whl

isa_model/inference/services/llm/helpers/llm_utils.py

@@ -95,34 +95,332 @@ class TextProcessor:
         return code_blocks
     
     @staticmethod
-    def extract_json_from_text(text: str) -> Optional[Dict[str, Any]]:
-        """从文本中提取JSON"""
-        # 尝试直接解析
-        try:
-            return json.loads(text)
-        except json.JSONDecodeError:
-            pass
+    def extract_json_from_text(text: str, schema: Optional[Dict] = None, repair_attempts: int = 3) -> Dict[str, Any]:
+        """Enhanced JSON extraction with validation and error recovery"""
+        result = {
+            "success": False,
+            "data": None,
+            "errors": [],
+            "method": None,
+            "repaired": False
+        }
         
-        # 尝试提取JSON代码块
-        json_pattern = r'```json\n(.*?)\n```'
-        match = re.search(json_pattern, text, re.DOTALL)
-        if match:
-            try:
-                return json.loads(match.group(1))
-            except json.JSONDecodeError:
-                pass
-        
-        # 尝试提取花括号内容
-        brace_pattern = r'\{.*\}'
-        match = re.search(brace_pattern, text, re.DOTALL)
-        if match:
+        extraction_methods = [
+            ("direct_parse", TextProcessor._try_direct_json_parse),
+            ("json_code_block", TextProcessor._try_json_code_block),
+            ("first_json_object", TextProcessor._try_first_json_object),
+            ("between_braces", TextProcessor._try_between_braces),
+            ("multiple_objects", TextProcessor._try_multiple_json_objects),
+            ("yaml_like", TextProcessor._try_yaml_like_parsing)
+        ]
+        
+        for method_name, extraction_func in extraction_methods:
             try:
-                return json.loads(match.group(0))
-            except json.JSONDecodeError:
-                pass
+                extracted_data = extraction_func(text)
+                if extracted_data is not None:
+                    # Validate against schema if provided
+                    if schema:
+                        validation_result = TextProcessor._validate_json_schema(extracted_data, schema)
+                        if validation_result["valid"]:
+                            result.update({
+                                "success": True,
+                                "data": extracted_data,
+                                "method": method_name
+                            })
+                            return result
+                        else:
+                            result["errors"].append(f"{method_name}: {validation_result['error']}")
+                    else:
+                        result.update({
+                            "success": True,
+                            "data": extracted_data,
+                            "method": method_name
+                        })
+                        return result
+            except Exception as e:
+                result["errors"].append(f"{method_name}: {str(e)}")
+                continue
+        
+        # If all methods failed, try repair attempts
+        if repair_attempts > 0:
+            repair_result = TextProcessor._attempt_json_repair(text, schema, repair_attempts)
+            if repair_result["success"]:
+                result.update(repair_result)
+                result["repaired"] = True
+                return result
+            else:
+                result["errors"].extend(repair_result["errors"])
+        
+        return result
+    
+    @staticmethod
+    def _try_direct_json_parse(text: str) -> Optional[Dict[str, Any]]:
+        """Try to parse text directly as JSON"""
+        text = text.strip()
+        if not text:
+            return None
+        return json.loads(text)
+    
+    @staticmethod
+    def _try_json_code_block(text: str) -> Optional[Dict[str, Any]]:
+        """Extract JSON from code blocks"""
+        patterns = [
+            r'```json\s*(.*?)\s*```',
+            r'```JSON\s*(.*?)\s*```',
+            r'```\s*(\{.*?\})\s*```'
+        ]
+        
+        for pattern in patterns:
+            match = re.search(pattern, text, re.DOTALL | re.IGNORECASE)
+            if match:
+                json_str = match.group(1).strip()
+                if json_str:
+                    return json.loads(json_str)
         
         return None
     
+    @staticmethod
+    def _try_first_json_object(text: str) -> Optional[Dict[str, Any]]:
+        """Find and parse the first JSON object in text"""
+        # Look for the first { and find its matching }
+        start_idx = text.find('{')
+        if start_idx == -1:
+            return None
+        
+        brace_count = 0
+        in_string = False
+        escape_next = False
+        
+        for i, char in enumerate(text[start_idx:], start_idx):
+            if escape_next:
+                escape_next = False
+                continue
+                
+            if char == '\\':
+                escape_next = True
+                continue
+                
+            if char == '"' and not escape_next:
+                in_string = not in_string
+                continue
+            
+            if not in_string:
+                if char == '{':
+                    brace_count += 1
+                elif char == '}':
+                    brace_count -= 1
+                    if brace_count == 0:
+                        json_str = text[start_idx:i+1]
+                        return json.loads(json_str)
+        
+        return None
+    
+    @staticmethod
+    def _try_between_braces(text: str) -> Optional[Dict[str, Any]]:
+        """Extract content between first { and last }"""
+        first_brace = text.find('{')
+        last_brace = text.rfind('}')
+        
+        if first_brace != -1 and last_brace != -1 and last_brace > first_brace:
+            json_str = text[first_brace:last_brace+1]
+            return json.loads(json_str)
+        
+        return None
+    
+    @staticmethod
+    def _try_multiple_json_objects(text: str) -> Optional[List[Dict[str, Any]]]:
+        """Try to extract multiple JSON objects"""
+        objects = []
+        remaining_text = text
+        
+        while True:
+            try:
+                obj = TextProcessor._try_first_json_object(remaining_text)
+                if obj is None:
+                    break
+                objects.append(obj)
+                
+                # Find where this object ends and continue
+                obj_str = json.dumps(obj)
+                obj_end = remaining_text.find('}')
+                if obj_end == -1:
+                    break
+                remaining_text = remaining_text[obj_end+1:]
+                
+            except:
+                break
+        
+        return objects if objects else None
+    
+    @staticmethod
+    def _try_yaml_like_parsing(text: str) -> Optional[Dict[str, Any]]:
+        """Try to parse YAML-like structures and convert to JSON"""
+        try:
+            # Simple YAML-like parsing for basic cases
+            lines = text.strip().split('\n')
+            result = {}
+            
+            for line in lines:
+                line = line.strip()
+                if ':' in line and not line.startswith('#'):
+                    key, value = line.split(':', 1)
+                    key = key.strip().strip('"\'')
+                    value = value.strip().strip('"\'')
+                    
+                    # Try to convert value to appropriate type
+                    if value.lower() in ['true', 'false']:
+                        value = value.lower() == 'true'
+                    elif value.isdigit():
+                        value = int(value)
+                    elif '.' in value and value.replace('.', '').isdigit():
+                        value = float(value)
+                    
+                    result[key] = value
+            
+            return result if result else None
+        except:
+            return None
+    
+    @staticmethod
+    def _validate_json_schema(data: Any, schema: Dict) -> Dict[str, Any]:
+        """Validate JSON data against a simple schema"""
+        try:
+            # Try jsonschema library first
+            try:
+                import jsonschema
+                jsonschema.validate(instance=data, schema=schema)
+                return {"valid": True, "error": None}
+            except ImportError:
+                # Fallback to basic validation
+                return TextProcessor._basic_schema_validation(data, schema)
+            except jsonschema.ValidationError as e:
+                return {"valid": False, "error": str(e)}
+                
+        except Exception as e:
+            return {"valid": False, "error": f"Schema validation error: {str(e)}"}
+    
+    @staticmethod
+    def _basic_schema_validation(data: Any, schema: Dict) -> Dict[str, Any]:
+        """Basic schema validation without jsonschema library"""
+        try:
+            if "type" in schema:
+                expected_type = schema["type"]
+                if expected_type == "object" and not isinstance(data, dict):
+                    return {"valid": False, "error": f"Expected object, got {type(data).__name__}"}
+                elif expected_type == "array" and not isinstance(data, list):
+                    return {"valid": False, "error": f"Expected array, got {type(data).__name__}"}
+                elif expected_type == "string" and not isinstance(data, str):
+                    return {"valid": False, "error": f"Expected string, got {type(data).__name__}"}
+                elif expected_type == "number" and not isinstance(data, (int, float)):
+                    return {"valid": False, "error": f"Expected number, got {type(data).__name__}"}
+                elif expected_type == "boolean" and not isinstance(data, bool):
+                    return {"valid": False, "error": f"Expected boolean, got {type(data).__name__}"}
+            
+            if "required" in schema and isinstance(data, dict):
+                for required_field in schema["required"]:
+                    if required_field not in data:
+                        return {"valid": False, "error": f"Missing required field: {required_field}"}
+            
+            return {"valid": True, "error": None}
+        except Exception as e:
+            return {"valid": False, "error": f"Basic validation error: {str(e)}"}
+    
+    @staticmethod
+    def _attempt_json_repair(text: str, schema: Optional[Dict], max_attempts: int) -> Dict[str, Any]:
+        """Attempt to repair malformed JSON"""
+        result = {"success": False, "data": None, "errors": []}
+        
+        repair_strategies = [
+            TextProcessor._repair_missing_quotes,
+            TextProcessor._repair_trailing_commas,
+            TextProcessor._repair_unescaped_quotes,
+            TextProcessor._repair_incomplete_objects
+        ]
+        
+        current_text = text
+        
+        for attempt in range(max_attempts):
+            for strategy_name, repair_func in [
+                ("missing_quotes", TextProcessor._repair_missing_quotes),
+                ("trailing_commas", TextProcessor._repair_trailing_commas),
+                ("unescaped_quotes", TextProcessor._repair_unescaped_quotes),
+                ("incomplete_objects", TextProcessor._repair_incomplete_objects)
+            ]:
+                try:
+                    repaired_text = repair_func(current_text)
+                    if repaired_text != current_text:
+                        # Try to parse the repaired text
+                        try:
+                            repaired_data = json.loads(repaired_text)
+                            
+                            # Validate if schema provided
+                            if schema:
+                                validation = TextProcessor._validate_json_schema(repaired_data, schema)
+                                if validation["valid"]:
+                                    result.update({
+                                        "success": True,
+                                        "data": repaired_data,
+                                        "method": f"repair_{strategy_name}"
+                                    })
+                                    return result
+                                else:
+                                    result["errors"].append(f"repair_{strategy_name}: {validation['error']}")
+                            else:
+                                result.update({
+                                    "success": True,
+                                    "data": repaired_data,
+                                    "method": f"repair_{strategy_name}"
+                                })
+                                return result
+                        except json.JSONDecodeError as e:
+                            result["errors"].append(f"repair_{strategy_name}: Still invalid JSON after repair - {str(e)}")
+                        
+                        current_text = repaired_text
+                except Exception as e:
+                    result["errors"].append(f"repair_{strategy_name}: Repair attempt failed - {str(e)}")
+        
+        return result
+    
+    @staticmethod
+    def _repair_missing_quotes(text: str) -> str:
+        """Add quotes around unquoted keys"""
+        # Simple regex to add quotes around keys
+        pattern = r'(\s*)([a-zA-Z_][a-zA-Z0-9_]*)\s*:'
+        return re.sub(pattern, r'\1"\2":', text)
+    
+    @staticmethod
+    def _repair_trailing_commas(text: str) -> str:
+        """Remove trailing commas"""
+        # Remove trailing commas before } or ]
+        text = re.sub(r',\s*([}\]])', r'\1', text)
+        return text
+    
+    @staticmethod
+    def _repair_unescaped_quotes(text: str) -> str:
+        """Escape unescaped quotes in strings"""
+        # This is a simplified approach - in practice this is quite complex
+        # Replace unescaped quotes that are clearly inside string values
+        return text.replace('\\"', '"').replace('"', '\\"').replace('\\\\"', '\\"')
+    
+    @staticmethod
+    def _repair_incomplete_objects(text: str) -> str:
+        """Try to complete incomplete JSON objects"""
+        text = text.strip()
+        
+        # Add missing closing braces
+        open_braces = text.count('{')
+        close_braces = text.count('}')
+        if open_braces > close_braces:
+            text += '}' * (open_braces - close_braces)
+        
+        # Add missing closing brackets
+        open_brackets = text.count('[')
+        close_brackets = text.count(']')
+        if open_brackets > close_brackets:
+            text += ']' * (open_brackets - close_brackets)
+        
+        return text
+    
     @staticmethod
     def split_into_sentences(text: str) -> List[str]:
         """将文本分割为句子"""