headroom-ai 0.2.13__py3-none-any.whl

headroom/transforms/content_detector.py

@@ -0,0 +1,335 @@
+"""Content type detection for multi-format compression.
+
+This module detects the type of tool output content to route it to the
+appropriate compressor. SmartCrusher handles JSON arrays, but coding tasks
+produce many other formats that need specialized handling.
+
+Supported content types:
+- JSON_ARRAY: Structured JSON data (existing SmartCrusher)
+- SOURCE_CODE: Python, JavaScript, TypeScript, Go, etc.
+- SEARCH_RESULTS: grep/ripgrep output (file:line:content)
+- BUILD_OUTPUT: Compiler, test, lint logs
+- GIT_DIFF: Unified diff format
+- PLAIN_TEXT: Generic text (fallback)
+"""
+
+from __future__ import annotations
+
+import json
+import re
+from dataclasses import dataclass
+from enum import Enum
+
+
+class ContentType(Enum):
+    """Types of content that can be compressed."""
+
+    JSON_ARRAY = "json_array"  # Existing SmartCrusher handles this
+    SOURCE_CODE = "source_code"  # Python, JS, TS, Go, Rust, etc.
+    SEARCH_RESULTS = "search"  # grep/ripgrep output
+    BUILD_OUTPUT = "build"  # Compiler, test, lint logs
+    GIT_DIFF = "diff"  # Unified diff format
+    PLAIN_TEXT = "text"  # Fallback
+
+
+@dataclass
+class DetectionResult:
+    """Result of content type detection."""
+
+    content_type: ContentType
+    confidence: float  # 0.0 to 1.0
+    metadata: dict  # Type-specific metadata (e.g., language for code)
+
+
+# Patterns for detection
+_SEARCH_RESULT_PATTERN = re.compile(
+    r"^[^\s:]+:\d+:"  # file:line: format (grep -n style)
+)
+
+_DIFF_HEADER_PATTERN = re.compile(r"^(diff --git|--- a/|@@\s+-\d+,\d+\s+\+\d+,\d+\s+@@)")
+
+_DIFF_CHANGE_PATTERN = re.compile(r"^[+-][^+-]")
+
+# Code patterns by language
+_CODE_PATTERNS = {
+    "python": [
+        re.compile(r"^\s*(def|class|import|from|async def)\s+\w+"),
+        re.compile(r"^\s*@\w+"),  # decorators
+        re.compile(r'^\s*"""'),  # docstrings
+        re.compile(r"^\s*if __name__\s*=="),
+    ],
+    "javascript": [
+        re.compile(r"^\s*(function|const|let|var|class|import|export)\s+"),
+        re.compile(r"^\s*(async\s+function|=>\s*\{)"),
+        re.compile(r"^\s*module\.exports"),
+    ],
+    "typescript": [
+        re.compile(r"^\s*(interface|type|enum|namespace)\s+\w+"),
+        re.compile(r":\s*(string|number|boolean|any|void)\b"),
+    ],
+    "go": [
+        re.compile(r"^\s*(func|type|package|import)\s+"),
+        re.compile(r"^\s*func\s+\([^)]+\)\s+\w+"),  # method
+    ],
+    "rust": [
+        re.compile(r"^\s*(fn|struct|enum|impl|mod|use|pub)\s+"),
+        re.compile(r"^\s*#\["),  # attributes
+    ],
+    "java": [
+        re.compile(r"^\s*(public|private|protected)\s+(class|interface|enum)"),
+        re.compile(r"^\s*@\w+"),  # annotations
+        re.compile(r"^\s*package\s+[\w.]+;"),
+    ],
+}
+
+# Log/build output patterns
+_LOG_PATTERNS = [
+    re.compile(r"\b(ERROR|FAIL|FAILED|FATAL|CRITICAL)\b", re.IGNORECASE),
+    re.compile(r"\b(WARN|WARNING)\b", re.IGNORECASE),
+    re.compile(r"\b(INFO|DEBUG|TRACE)\b", re.IGNORECASE),
+    re.compile(r"^\s*\d{4}-\d{2}-\d{2}"),  # timestamp
+    re.compile(r"^\s*\[\d{2}:\d{2}:\d{2}\]"),  # time format
+    re.compile(r"^={3,}|^-{3,}"),  # separators
+    re.compile(r"^\s*PASSED|^\s*FAILED|^\s*SKIPPED"),  # test results
+    re.compile(r"^npm ERR!|^yarn error|^cargo error"),  # build tools
+    re.compile(r"Traceback \(most recent call last\)"),  # Python traceback
+    re.compile(r"^\s*at\s+[\w.$]+\("),  # JS/Java stack trace
+]
+
+
+def detect_content_type(content: str) -> DetectionResult:
+    """Detect the type of content for appropriate compression.
+
+    Args:
+        content: The content to analyze.
+
+    Returns:
+        DetectionResult with type, confidence, and metadata.
+
+    Examples:
+        >>> result = detect_content_type('[{"id": 1}, {"id": 2}]')
+        >>> result.content_type
+        ContentType.JSON_ARRAY
+
+        >>> result = detect_content_type('src/main.py:42:def process():')
+        >>> result.content_type
+        ContentType.SEARCH_RESULTS
+    """
+    if not content or not content.strip():
+        return DetectionResult(ContentType.PLAIN_TEXT, 0.0, {})
+
+    # 1. Try JSON first (highest priority for SmartCrusher compatibility)
+    json_result = _try_detect_json(content)
+    if json_result:
+        return json_result
+
+    # 2. Check for diff (very distinctive patterns)
+    diff_result = _try_detect_diff(content)
+    if diff_result and diff_result.confidence >= 0.7:
+        return diff_result
+
+    # 3. Check for search results (file:line: format)
+    search_result = _try_detect_search(content)
+    if search_result and search_result.confidence >= 0.6:
+        return search_result
+
+    # 4. Check for build/log output
+    log_result = _try_detect_log(content)
+    if log_result and log_result.confidence >= 0.5:
+        return log_result
+
+    # 5. Check for source code
+    code_result = _try_detect_code(content)
+    if code_result and code_result.confidence >= 0.5:
+        return code_result
+
+    # 6. Fallback to plain text
+    return DetectionResult(ContentType.PLAIN_TEXT, 0.5, {})
+
+
+def _try_detect_json(content: str) -> DetectionResult | None:
+    """Try to detect JSON array content."""
+    content = content.strip()
+
+    # Quick check: must start with [ for array
+    if not content.startswith("["):
+        return None
+
+    try:
+        parsed = json.loads(content)
+        if isinstance(parsed, list):
+            # Check if it's a list of dicts (SmartCrusher compatible)
+            if parsed and all(isinstance(item, dict) for item in parsed):
+                return DetectionResult(
+                    ContentType.JSON_ARRAY,
+                    1.0,
+                    {"item_count": len(parsed), "is_dict_array": True},
+                )
+            # It's a list but not of dicts
+            return DetectionResult(
+                ContentType.JSON_ARRAY,
+                0.8,
+                {"item_count": len(parsed), "is_dict_array": False},
+            )
+    except json.JSONDecodeError:
+        pass
+
+    return None
+
+
+def _try_detect_diff(content: str) -> DetectionResult | None:
+    """Try to detect git diff format."""
+    lines = content.split("\n")[:50]  # Check first 50 lines
+
+    header_matches = 0
+    change_matches = 0
+
+    for line in lines:
+        if _DIFF_HEADER_PATTERN.match(line):
+            header_matches += 1
+        if _DIFF_CHANGE_PATTERN.match(line):
+            change_matches += 1
+
+    if header_matches == 0:
+        return None
+
+    # High confidence if we see diff headers
+    confidence = min(1.0, 0.5 + (header_matches * 0.2) + (change_matches * 0.05))
+
+    return DetectionResult(
+        ContentType.GIT_DIFF,
+        confidence,
+        {"header_matches": header_matches, "change_lines": change_matches},
+    )
+
+
+def _try_detect_search(content: str) -> DetectionResult | None:
+    """Try to detect grep/ripgrep search results."""
+    lines = content.split("\n")[:100]  # Check first 100 lines
+    if not lines:
+        return None
+
+    matching_lines = 0
+    for line in lines:
+        if line.strip() and _SEARCH_RESULT_PATTERN.match(line):
+            matching_lines += 1
+
+    if matching_lines == 0:
+        return None
+
+    # Calculate confidence based on proportion of matching lines
+    non_empty_lines = sum(1 for line in lines if line.strip())
+    if non_empty_lines == 0:
+        return None
+
+    ratio = matching_lines / non_empty_lines
+
+    # Need at least 30% of lines to match the pattern
+    if ratio < 0.3:
+        return None
+
+    confidence = min(1.0, 0.4 + (ratio * 0.6))
+
+    return DetectionResult(
+        ContentType.SEARCH_RESULTS,
+        confidence,
+        {"matching_lines": matching_lines, "total_lines": non_empty_lines},
+    )
+
+
+def _try_detect_log(content: str) -> DetectionResult | None:
+    """Try to detect build/log output."""
+    lines = content.split("\n")[:200]  # Check first 200 lines
+    if not lines:
+        return None
+
+    pattern_matches = 0
+    error_matches = 0
+
+    for line in lines:
+        for i, pattern in enumerate(_LOG_PATTERNS):
+            if pattern.search(line):
+                pattern_matches += 1
+                if i < 2:  # ERROR or WARN patterns
+                    error_matches += 1
+                break  # One pattern per line is enough
+
+    if pattern_matches == 0:
+        return None
+
+    non_empty_lines = sum(1 for line in lines if line.strip())
+    if non_empty_lines == 0:
+        return None
+
+    ratio = pattern_matches / non_empty_lines
+
+    # Need at least 10% of lines to match log patterns
+    if ratio < 0.1:
+        return None
+
+    confidence = min(1.0, 0.3 + (ratio * 0.5) + (error_matches * 0.05))
+
+    return DetectionResult(
+        ContentType.BUILD_OUTPUT,
+        confidence,
+        {
+            "pattern_matches": pattern_matches,
+            "error_matches": error_matches,
+            "total_lines": non_empty_lines,
+        },
+    )
+
+
+def _try_detect_code(content: str) -> DetectionResult | None:
+    """Try to detect source code and identify language."""
+    lines = content.split("\n")[:100]  # Check first 100 lines
+    if not lines:
+        return None
+
+    language_scores: dict[str, int] = {}
+
+    for line in lines:
+        for lang, patterns in _CODE_PATTERNS.items():
+            for pattern in patterns:
+                if pattern.match(line):
+                    language_scores[lang] = language_scores.get(lang, 0) + 1
+                    break  # One pattern per language per line
+
+    if not language_scores:
+        return None
+
+    # Find best matching language
+    best_lang = max(language_scores, key=lambda k: language_scores[k])
+    best_score = language_scores[best_lang]
+
+    # Need at least 3 pattern matches to be confident
+    if best_score < 3:
+        return None
+
+    non_empty_lines = sum(1 for line in lines if line.strip())
+    ratio = best_score / max(non_empty_lines, 1)
+
+    confidence = min(1.0, 0.4 + (ratio * 0.4) + (best_score * 0.02))
+
+    return DetectionResult(
+        ContentType.SOURCE_CODE,
+        confidence,
+        {"language": best_lang, "pattern_matches": best_score},
+    )
+
+
+def is_json_array_of_dicts(content: str) -> bool:
+    """Quick check if content is a JSON array of dictionaries.
+
+    This is the format SmartCrusher can handle natively.
+
+    Args:
+        content: The content to check.
+
+    Returns:
+        True if content is a JSON array where all items are dicts.
+    """
+    result = detect_content_type(content)
+    return result.content_type == ContentType.JSON_ARRAY and result.metadata.get(
+        "is_dict_array", False
+    )