agent-api-server 2.1.7__py3-none-any.whl

agent_api_server/shared/detect_message.py

@@ -0,0 +1,410 @@
+from typing import Literal, List, Tuple, Dict, Set
+import re
+import json
+from dataclasses import dataclass
+from collections import defaultdict
+
+ContentType = Literal["markdown", "json", "html", "xml", "python", "yaml", "text"]
+
+
+@dataclass
+class ContentTypeDetector:
+    name: ContentType
+    detector: callable
+    priority: int = 0
+    confidence_threshold: float = 0.7
+    exclusive: bool = False
+
+def detect_content_type(content: str) -> ContentType:
+    content = content.strip()
+    if not content:
+        return "text"
+
+    results: List[Tuple[ContentType, float]] = []
+    detectors = sorted(CONTENT_TYPE_DETECTORS, key=lambda x: (-x.priority, x.name))
+
+    for detector in detectors:
+        try:
+            confidence = detector.detector(content)
+            if confidence > 0:
+                results.append((detector.name, confidence))
+                # If this type is exclusive and meets threshold, return immediately
+                if detector.exclusive and confidence >= detector.confidence_threshold:
+                    return detector.name
+        except Exception:
+            continue
+
+    if not results:
+        return "text"
+
+    results = _adjust_confidences(content, results)
+
+    valid_results = [
+        (name, conf) for name, conf in results
+        if conf >= next(d.confidence_threshold for d in detectors if d.name == name)
+    ]
+
+    if not valid_results:
+        return "text"
+
+    best_match = max(valid_results, key=lambda x: (x[1], -next(
+        i for i, d in enumerate(detectors) if d.name == x[0]
+    )))
+
+    if _verify_content_type(content, best_match[0]):
+        return best_match[0]
+    elif len(valid_results) > 1:
+        second_best = max(
+            [r for r in valid_results if r[0] != best_match[0]],
+            key=lambda x: (x[1], -next(i for i, d in enumerate(detectors) if d.name == x[0])),
+            default=None
+        )
+        if second_best and _verify_content_type(content, second_best[0]):
+            return second_best[0]
+
+    return "text"
+
+
+def _adjust_confidences(content: str, results: List[Tuple[ContentType, float]]) -> List[Tuple[ContentType, float]]:
+    """Apply content-specific adjustments to confidence scores."""
+    type_scores = {name: conf for name, conf in results}
+
+    # Penalize HTML if it looks like template syntax
+    if "html" in type_scores and any(
+            re.search(p, content) for p in [r'\{\{.*\}\}', r'{%.*%}', r'<\?php']):
+        type_scores["html"] *= 0.7
+
+    # Boost JSON if it's the only structured format detected
+    if "json" in type_scores and len(type_scores) == 1:
+        type_scores["json"] = min(1.0, type_scores["json"] * 1.1)
+
+    # Penalize YAML if it looks like plain text with colons
+    if "yaml" in type_scores and not any(
+            re.search(p, content) for p in [r'^\s*-\s+', r'^\s*\w+:\s*\n\s+', r'^---\s*$']
+    ):
+        type_scores["yaml"] *= 0.6
+
+    # Penalize Markdown if it has too many consecutive non-MD lines
+    if "markdown" in type_scores:
+        lines = content.splitlines()
+        md_lines = sum(1 for line in lines if any(
+            re.search(p, line) for p in MD_PATTERNS
+        ))
+        if md_lines / len(lines) < 0.2:
+            type_scores["markdown"] *= 0.5
+
+    return list(type_scores.items())
+
+
+def _verify_content_type(content: str, content_type: ContentType) -> bool:
+    """Perform additional verification for the detected content type."""
+    if content_type == "json":
+        try:
+            json.loads(content)
+            return True
+        except:
+            return False
+    elif content_type == "python":
+        return _verify_python(content)
+    elif content_type == "html":
+        return _verify_html(content)
+    elif content_type == "xml":
+        return _verify_xml(content)
+    return True
+
+
+def _verify_python(content: str) -> bool:
+    """Additional verification for Python code."""
+    lines = [line for line in content.splitlines() if line.strip()]
+    if not lines:
+        return False
+
+    # Check for valid Python syntax structures
+    def_count = len(re.findall(r'^\s*def\s+\w+\s*$', content, re.MULTILINE))
+    class_count = len(re.findall(r'^\s*class\s+\w+', content, re.MULTILINE))
+    import_count = len(re.findall(r'^\s*import\s+\w+', content, re.MULTILINE))
+
+    return (def_count + class_count + import_count) >= 1
+
+
+def _verify_html(content: str) -> bool:
+    """Additional verification for HTML."""
+    open_tags = re.findall(r'<([a-z]+)[^>]*>', content.lower())
+    close_tags = re.findall(r'</([a-z]+)>', content.lower())
+
+    if not open_tags and not close_tags:
+        return False
+
+    # Check if most opened tags are closed
+    tag_counts = defaultdict(int)
+    for tag in open_tags:
+        tag_counts[tag] += 1
+    for tag in close_tags:
+        tag_counts[tag] -= 1
+
+    properly_closed = sum(1 for count in tag_counts.values() if count <= 0)
+    return properly_closed / len(tag_counts) > 0.5 if tag_counts else False
+
+
+def _verify_xml(content: str) -> bool:
+    """Additional verification for XML."""
+    if '<?xml version=' in content[:100].lower():
+        return True
+
+    # Check for balanced tags
+    tags = re.findall(r'<(/?)([a-z]+)[^>]*>', content.lower())
+    stack = []
+    for is_close, tag in tags:
+        if not is_close:
+            stack.append(tag)
+        else:
+            if not stack or stack[-1] != tag:
+                return False
+            stack.pop()
+    return not stack
+
+
+# Predefined patterns for various content types
+MD_PATTERNS = [
+    r'^#+\s', r'^[-*]\s', r'^\d+\.\s', r'^>\s',
+    r'$$.*$$\(.*$', r'\*\*.*\*\*|__.*__',
+    r'\*[^*]+\*|_[^_]+_', r'^```', r'^\|.*\|.*\|$',
+    r'^---$|^===', r'^`[^`]+`', r'!$$.*$$$.*$'
+]
+
+PYTHON_KEYWORDS = {
+    'def ', 'class ', 'import ', 'from ', 'try:', 'except:',
+    'if ', 'else:', 'elif ', 'for ', 'while ', 'with ',
+    'return ', 'yield ', 'async ', 'await ', 'raise ', 'lambda ',
+    'nonlocal ', 'global ', 'pass ', 'break ', 'continue '
+}
+
+PYTHON_CONSTRUCTS = [
+    r'^\s*@[\w\.]+', r'^\s*def\s+\w+\s*$', r'^\s*class\s+\w+',
+    r'^\s*[\w_]+\s*=', r'^\s*[\w_]+\s*:\s*\w+', r'^\s*[\w_]+\s*=[^=]',
+    r'^\s*async\s+def\s+', r'^\s*await\s+', r'^\s*\(\s*$\s*:'
+]
+
+HTML_PATTERNS = [
+    (r'<!doctype\s+html>', 0.3), (r'<html[\s>]', 0.2), (r'<head[\s>]', 0.2),
+    (r'<body[\s>]', 0.2), (r'<div[\s>]', 0.15), (r'<p[\s>]', 0.15),
+    (r'<a\s+href=', 0.2), (r'<img\s+src=', 0.2), (r'<script[\s>]', 0.2),
+    (r'<style[\s>]', 0.2), (r'</(html|head|body|div|p|a|img|script|style)>', 0.3)
+]
+
+XML_PATTERNS = [
+    (r'<\?xml\s+version=', 1.0), (r'^<[^!?][^>]*>.*</[^>]+>$', 0.9),
+    (r'<[^/][^>]*/>', 0.7), (r'<(/?)(\w+)[^>]*>', 0.5)
+]
+
+YAML_PATTERNS = [
+    (r'^\s*[\w-]+\s*:', 0.5), (r'^\s*-\s*\w+', 0.4), (r'^---$', 0.3),
+    (r'^\s*[\w-]+\s*:\s*\|', 0.4), (r'^\s*[\w-]+\s*:\s*>\s*$', 0.4),
+    (r'^\s*#', 0.1), (r'^\s*\w+:\s*\n\s+', 0.3)
+]
+
+
+# Detector implementations with improved accuracy
+
+def _is_valid_json(content: str) -> float:
+    """Enhanced JSON detector with better pattern matching."""
+    content = content.strip()
+    if not content:
+        return 0.0
+
+    # Quick check for JSON structure
+    if not (content.startswith(('{', '[')) and content.endswith(('}', ']'))):
+        return 0.0
+
+    # Check for common JSON patterns
+    json_patterns = [
+        r'^\s*\{\s*".*"\s*:\s*[^,]+(,\s*".*"\s*:\s*[^,]+)*\s*\}\s*$',
+        r'^\s*$$\s*[^,]*(,\s*[^,]+)*\s*$$\s*$',
+        r'"\w+"\s*:\s*(null|true|false|"[^"]*"|\d+\.?\d*)'
+    ]
+
+    if not any(re.search(p, content, re.DOTALL) for p in json_patterns):
+        return 0.0
+
+    try:
+        json.loads(content)
+        return 1.0
+    except (ValueError, TypeError, json.JSONDecodeError):
+        return 0.0
+
+
+def _looks_like_python(content: str) -> float:
+    """Enhanced Python detector with better syntax analysis."""
+    lines = [line for line in content.splitlines() if line.strip()]
+    if not lines:
+        return 0.0
+
+    # Check for shebang
+    if lines[0].startswith('#!') and 'python' in lines[0].lower():
+        return 1.0
+
+    # Count Python-specific constructs
+    keyword_count = sum(
+        1 for line in lines
+        if any(kw in line for kw in PYTHON_KEYWORDS)
+    )
+    construct_count = sum(
+        1 for line in lines
+        if any(re.search(p, line) for p in PYTHON_CONSTRUCTS)
+    )
+
+    # Check indentation consistency
+    indent_consistent = True
+    indent_levels = set()
+    for line in lines:
+        if line.strip().startswith('#'):
+            continue
+        indent = len(line) - len(line.lstrip())
+        indent_levels.add(indent)
+        if len(indent_levels) > 3:  # More than 3 indent levels is suspicious
+            indent_consistent = False
+            break
+
+    # Calculate confidence
+    score = 0.0
+    if keyword_count > 0:
+        score += min(0.5, keyword_count * 0.15)
+    if construct_count > 0:
+        score += min(0.4, construct_count * 0.12)
+    if indent_consistent and len(indent_levels) > 1:
+        score += 0.2
+
+    return min(1.0, score)
+
+
+def _looks_like_html(content: str) -> float:
+    """Enhanced HTML detector with better tag analysis."""
+    content = content.lower()
+
+    # Check for HTML5 doctype
+    if re.search(r'<!doctype\s+html>', content):
+        return 1.0
+
+    # Calculate score based on patterns
+    score = sum(
+        weight for pattern, weight in HTML_PATTERNS
+        if re.search(pattern, content)
+    )
+
+    # Check tag balance
+    open_tags = re.findall(r'<([a-z]+)[^>]*>', content)
+    close_tags = re.findall(r'</([a-z]+)>', content)
+
+    if open_tags and close_tags:
+        tag_balance = min(len(open_tags), len(close_tags)) / max(len(open_tags), len(close_tags), 1)
+        score += min(0.3, tag_balance * 0.5)
+
+    return min(1.0, score)
+
+
+def _looks_like_xml(content: str) -> float:
+    """Enhanced XML detector with better structure analysis."""
+    content = content.strip()
+
+    if '<?xml ' in content[:100].lower():
+        return 1.0
+
+    # Check for root element
+    root_elements = re.findall(r'<([^!?][^>]*)>', content[:500])
+    if not root_elements:
+        return 0.0
+
+    # Calculate score
+    score = sum(
+        weight for pattern, weight in XML_PATTERNS
+        if re.search(pattern, content)
+    )
+
+    # Check for balanced tags
+    tags = re.findall(r'<(/?)(\w+)[^>]*>', content)
+    if tags:
+        stack = []
+        balanced = True
+        for is_close, tag in tags:
+            if not is_close:
+                stack.append(tag)
+            else:
+                if not stack or stack[-1] != tag:
+                    balanced = False
+                    break
+                stack.pop()
+        if balanced and not stack:
+            score += 0.3
+
+    return min(1.0, score)
+
+
+def _looks_like_markdown(content: str) -> float:
+    """Check if content looks like Markdown with confidence score."""
+    lines = [line for line in content.splitlines() if line.strip()]
+    if not lines:
+        return 0.0
+
+    md_patterns = [
+        (r'^#+\s', 0.3),  # Headers
+        (r'^[-*]\s', 0.2),  # Unordered lists
+        (r'^\d+\.\s', 0.2),  # Ordered lists
+        (r'^>\s', 0.2),  # Blockquotes
+        (r'$$.*?$$$.*?$', 0.3),  # Links - 修正这里
+        (r'\*\*.*?\*\*|__.*?__', 0.2),  # Bold
+        (r'\*.*?\*|_.*?_', 0.2),  # Italic
+        (r'^```', 0.3),  # Code blocks
+        (r'^\|.*\|.*\|$', 0.3),  # Tables
+        (r'^---$|^===', 0.2),  # Horizontal rules
+        (r'^`[^`]+`', 0.2),  # Inline code
+        (r'!$$.*?$$$.*?$', 0.3),  # Images - 修正这里
+    ]
+
+    md_lines = 0
+    total_lines = len(lines)
+
+    for line in lines:
+        for pattern, weight in md_patterns:
+            if re.search(pattern, line):
+                md_lines += 1
+                break
+
+    # Calculate confidence based on percentage of markdown lines
+    ratio = md_lines / total_lines
+    if ratio >= 0.3:  # At least 30% of lines are markdown
+        return min(1.0, ratio * 1.5)  # Scale to 0-1 range
+
+    return 0.0
+
+
+def _looks_like_yaml(content: str) -> float:
+    """Enhanced YAML detector with better structure analysis."""
+    lines = [line for line in content.splitlines() if line.strip()]
+    if not lines:
+        return 0.0
+
+    # Calculate score based on patterns
+    score = sum(
+        weight for pattern, weight in YAML_PATTERNS
+        if any(re.search(pattern, line) for line in lines)
+    )
+
+    # Check for common YAML structures
+    has_documents = any(re.search(r'^---\s*$', line) for line in lines)
+    has_nested = any(re.search(r'^\s*\w+:\s*\n\s+\w', line) for line in lines)
+    has_lists = any(re.search(r'^\s*-\s+', line) for line in lines)
+
+    if has_documents or has_nested or has_lists:
+        score = min(1.0, score * 1.2)
+
+    return min(1.0, score)
+
+# Configured detectors with optimized thresholds
+CONTENT_TYPE_DETECTORS: List[ContentTypeDetector] = [
+    ContentTypeDetector("json", _is_valid_json, priority=100, confidence_threshold=0.95, exclusive=True),
+    ContentTypeDetector("yaml", _looks_like_yaml, priority=90, confidence_threshold=0.8),
+    ContentTypeDetector("python", _looks_like_python, priority=80, confidence_threshold=0.75),
+    ContentTypeDetector("html", _looks_like_html, priority=70, confidence_threshold=0.7),
+    ContentTypeDetector("xml", _looks_like_xml, priority=60, confidence_threshold=0.8),
+    ContentTypeDetector("markdown", _looks_like_markdown, priority=40, confidence_threshold=0.65),
+]