emergent-translator 1.1.0__py3-none-any.whl

emergent_translator/claude_compression.py

@@ -0,0 +1,375 @@
+"""
+Claude API Text-Level Compression
+
+Reuses the existing COMMON_KEYS/COMMON_VALUES and AdaptiveCodebook to generate
+text short codes that save tokens in Claude API conversations.
+
+Before: {"task_type": "analyze", "priority": "high", "status": "pending"} (~18 tokens)
+After:  {"tt": "ANL", "pr": "HGH", "st": "PND"} (~12 tokens) + ~250 token legend (one-time)
+
+The legend pays for itself after ~40 compressed field occurrences.
+
+Usage:
+    from emergent_translator.claude_compression import (
+        TextCodebook, ClaudeCompressor, compress, decompress,
+    )
+
+    cb = TextCodebook.from_common_tables()
+    compressed = compress({"task_type": "analyze", "priority": "high"}, cb)
+    original = decompress(compressed, cb)
+"""
+
+import re
+from dataclasses import dataclass, field
+from typing import Any, Dict, List, Optional, Tuple
+
+
+# ---------------------------------------------------------------------------
+# Private helpers
+# ---------------------------------------------------------------------------
+
+def _generate_candidate(word: str) -> str:
+    """Produce a 1-3 char short code candidate for *word*.
+
+    Strategy:
+    - underscore words -> initials (task_type -> tt)
+    - short words (<=3 chars) -> pass through
+    - longer words -> first char + consonants (trimmed to 3)
+    """
+    if "_" in word:
+        return "".join(part[0] for part in word.split("_") if part)
+    if len(word) <= 3:
+        return word
+    consonants = [ch for ch in word[1:] if ch not in "aeiou"]
+    if consonants:
+        return (word[0] + "".join(consonants))[:3]
+    return word[:3]
+
+
+def _build_mapping(names: List[str], is_key: bool) -> Dict[str, str]:
+    """Generate unique short codes for a list of names.
+
+    Keys -> lowercase codes, values -> UPPERCASE codes (namespace separation).
+    Deduplicates with fallback strategies.
+    """
+    mapping: Dict[str, str] = {}
+    used_codes: set = set()
+
+    for name in names:
+        candidate = _generate_candidate(name.lower())
+
+        # Apply case convention
+        if is_key:
+            candidate = candidate.lower()
+        else:
+            candidate = candidate.upper()
+
+        # Deduplication fallbacks
+        if candidate in used_codes:
+            # Fallback 1: first 2 chars
+            fb = name.lower()[:2]
+            fb = fb.lower() if is_key else fb.upper()
+            if fb not in used_codes and fb != candidate:
+                candidate = fb
+            else:
+                # Fallback 2: first + last consonant
+                consonants = [ch for ch in name.lower() if ch not in "aeiou_"]
+                if len(consonants) >= 2:
+                    fb2 = consonants[0] + consonants[-1]
+                    fb2 = fb2.lower() if is_key else fb2.upper()
+                    if fb2 not in used_codes:
+                        candidate = fb2
+                        # fall through to numeric if still colliding
+
+                # Fallback 3: numeric suffix
+                if candidate in used_codes:
+                    base = candidate
+                    for i in range(2, 100):
+                        attempt = f"{base}{i}"
+                        if attempt not in used_codes:
+                            candidate = attempt
+                            break
+
+        mapping[name] = candidate
+        used_codes.add(candidate)
+
+    return mapping
+
+
+# ---------------------------------------------------------------------------
+# TextCodebook
+# ---------------------------------------------------------------------------
+
+@dataclass
+class TextCodebook:
+    """Bidirectional mapping between full names and short text codes."""
+
+    keys: Dict[str, str]           # full_key -> short_code
+    values: Dict[str, str]         # full_value -> SHORT_CODE
+    keys_rev: Dict[str, str] = field(default_factory=dict)   # short_code -> full_key
+    values_rev: Dict[str, str] = field(default_factory=dict)  # SHORT_CODE -> full_value
+
+    def __post_init__(self):
+        if not self.keys_rev:
+            self.keys_rev = {v: k for k, v in self.keys.items()}
+        if not self.values_rev:
+            self.values_rev = {v: k for k, v in self.values.items()}
+
+    @classmethod
+    def from_common_tables(cls) -> "TextCodebook":
+        """Build from existing COMMON_KEYS/COMMON_VALUES tables."""
+        from .batch_encoder import COMMON_KEYS, COMMON_VALUES
+
+        # Deduplicate aliases: keep the first (longest) name per byte id
+        seen_key_ids: Dict[int, str] = {}
+        for name, byte_id in COMMON_KEYS.items():
+            if byte_id not in seen_key_ids or len(name) > len(seen_key_ids[byte_id]):
+                seen_key_ids[byte_id] = name
+        unique_keys = list(seen_key_ids.values())
+
+        seen_val_ids: Dict[int, str] = {}
+        for name, byte_id in COMMON_VALUES.items():
+            if byte_id not in seen_val_ids or len(name) > len(seen_val_ids[byte_id]):
+                seen_val_ids[byte_id] = name
+        unique_values = list(seen_val_ids.values())
+
+        keys_map = _build_mapping(unique_keys, is_key=True)
+        values_map = _build_mapping(unique_values, is_key=False)
+
+        return cls(keys=keys_map, values=values_map)
+
+    @classmethod
+    def from_adaptive(cls, codebook) -> "TextCodebook":
+        """Build from an AdaptiveCodebook's active version.
+
+        Args:
+            codebook: An AdaptiveCodebook instance.
+        """
+        active = codebook.get_active()
+
+        # Deduplicate aliases (same logic as from_common_tables)
+        seen_key_ids: Dict[int, str] = {}
+        for name, byte_id in active.keys.items():
+            if byte_id not in seen_key_ids or len(name) > len(seen_key_ids[byte_id]):
+                seen_key_ids[byte_id] = name
+        unique_keys = list(seen_key_ids.values())
+
+        seen_val_ids: Dict[int, str] = {}
+        for name, byte_id in active.values.items():
+            if byte_id not in seen_val_ids or len(name) > len(seen_val_ids[byte_id]):
+                seen_val_ids[byte_id] = name
+        unique_values = list(seen_val_ids.values())
+
+        keys_map = _build_mapping(unique_keys, is_key=True)
+        values_map = _build_mapping(unique_values, is_key=False)
+
+        return cls(keys=keys_map, values=values_map)
+
+    @classmethod
+    def from_dicts(cls, keys_map: Dict[str, str], values_map: Dict[str, str]) -> "TextCodebook":
+        """Create from explicit mappings."""
+        return cls(keys=dict(keys_map), values=dict(values_map))
+
+    def legend(self) -> str:
+        """Compact legend string for inclusion in a system prompt.
+
+        Format: [EL]Keys:tt=task_type st=status...|Vals:ANL=analyze PND=pending...[/EL]
+        """
+        key_parts = " ".join(f"{code}={name}" for name, code in sorted(self.keys.items()))
+        val_parts = " ".join(f"{code}={name}" for name, code in sorted(self.values.items()))
+        return f"[EL]Keys:{key_parts}|Vals:{val_parts}[/EL]"
+
+
+# ---------------------------------------------------------------------------
+# Free functions: compress / decompress
+# ---------------------------------------------------------------------------
+
+_default_codebook: Optional[TextCodebook] = None
+
+
+def _get_default_codebook() -> TextCodebook:
+    global _default_codebook
+    if _default_codebook is None:
+        _default_codebook = TextCodebook.from_common_tables()
+    return _default_codebook
+
+
+def compress(data: Any, codebook: Optional[TextCodebook] = None) -> Any:
+    """Recursively replace dict keys and string values with short codes.
+
+    - Dict keys are looked up in codebook.keys (case-insensitive).
+    - String values are looked up in codebook.values (case-insensitive).
+    - Ints, floats, bools, None pass through unchanged.
+    - Unknown keys/values pass through unchanged.
+    """
+    cb = codebook if codebook is not None else _get_default_codebook()
+    return _compress_recursive(data, cb)
+
+
+def _compress_recursive(data: Any, cb: TextCodebook) -> Any:
+    if isinstance(data, dict):
+        result = {}
+        for k, v in data.items():
+            new_key = cb.keys.get(k.lower(), k) if isinstance(k, str) else k
+            result[new_key] = _compress_recursive(v, cb)
+        return result
+    if isinstance(data, list):
+        return [_compress_recursive(item, cb) for item in data]
+    if isinstance(data, str):
+        return cb.values.get(data.lower(), data)
+    # int, float, bool, None — pass through
+    return data
+
+
+def decompress(data: Any, codebook: Optional[TextCodebook] = None) -> Any:
+    """Reverse the short-code mapping back to full names.
+
+    Unknown codes pass through unchanged.
+    """
+    cb = codebook if codebook is not None else _get_default_codebook()
+    return _decompress_recursive(data, cb)
+
+
+def _decompress_recursive(data: Any, cb: TextCodebook) -> Any:
+    if isinstance(data, dict):
+        result = {}
+        for k, v in data.items():
+            new_key = cb.keys_rev.get(k, k) if isinstance(k, str) else k
+            result[new_key] = _decompress_recursive(v, cb)
+        return result
+    if isinstance(data, list):
+        return [_decompress_recursive(item, cb) for item in data]
+    if isinstance(data, str):
+        return cb.values_rev.get(data, data)
+    return data
+
+
+# ---------------------------------------------------------------------------
+# Token estimation
+# ---------------------------------------------------------------------------
+
+def estimate_tokens(text: str) -> int:
+    """Estimate token count for *text*.
+
+    Uses tiktoken cl100k_base if available, otherwise chars//4 heuristic.
+    """
+    try:
+        import tiktoken
+        enc = tiktoken.get_encoding("cl100k_base")
+        return len(enc.encode(text))
+    except Exception:
+        return max(1, len(text) // 4)
+
+
+# ---------------------------------------------------------------------------
+# ClaudeCompressor
+# ---------------------------------------------------------------------------
+
+class ClaudeCompressor:
+    """High-level wrapper for compressing Claude API messages."""
+
+    def __init__(self, codebook: Optional[TextCodebook] = None):
+        self._codebook = codebook if codebook is not None else TextCodebook.from_common_tables()
+
+    @property
+    def codebook(self) -> TextCodebook:
+        return self._codebook
+
+    def compress_messages(self, messages: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
+        """Compress content in Claude API message format, preserving ``role``."""
+        result = []
+        for msg in messages:
+            new_msg = dict(msg)
+            if "content" in new_msg:
+                new_msg["content"] = compress(new_msg["content"], self._codebook)
+            result.append(new_msg)
+        return result
+
+    def decompress_messages(self, messages: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
+        """Decompress content in Claude API message format."""
+        result = []
+        for msg in messages:
+            new_msg = dict(msg)
+            if "content" in new_msg:
+                new_msg["content"] = decompress(new_msg["content"], self._codebook)
+            result.append(new_msg)
+        return result
+
+    def system_prompt_prefix(self) -> str:
+        """Return legend + instruction text suitable for prepending to a system prompt."""
+        legend = self._codebook.legend()
+        instruction = (
+            "When responding with structured data (JSON/dicts), "
+            "use the [EL] short codes defined above for all keys and values that have mappings. "
+            "Pass through any keys or values not in the legend unchanged."
+        )
+        return f"{legend}\n{instruction}"
+
+    def wrap_tool_result(self, result: Any) -> Any:
+        """Compress a single tool result (dict, list, or string)."""
+        return compress(result, self._codebook)
+
+    def estimate_savings(self, messages: List[Dict[str, Any]]) -> Dict[str, Any]:
+        """Estimate token savings from compression.
+
+        Returns dict with original_tokens, compressed_tokens, legend_tokens,
+        net_savings, savings_pct.
+        """
+        import json as _json
+
+        original_text = _json.dumps(messages)
+        compressed_msgs = self.compress_messages(messages)
+        compressed_text = _json.dumps(compressed_msgs)
+        legend_text = self.system_prompt_prefix()
+
+        original_tokens = estimate_tokens(original_text)
+        compressed_tokens = estimate_tokens(compressed_text)
+        legend_tokens = estimate_tokens(legend_text)
+
+        net_savings = original_tokens - compressed_tokens - legend_tokens
+        savings_pct = (net_savings / original_tokens * 100) if original_tokens > 0 else 0.0
+
+        return {
+            "original_tokens": original_tokens,
+            "compressed_tokens": compressed_tokens,
+            "legend_tokens": legend_tokens,
+            "net_savings": net_savings,
+            "savings_pct": savings_pct,
+        }
+
+
+# ---------------------------------------------------------------------------
+# wrap_api_call (optional convenience)
+# ---------------------------------------------------------------------------
+
+def wrap_api_call(
+    messages: List[Dict[str, Any]],
+    system: str = "",
+    codebook: Optional[TextCodebook] = None,
+    **kwargs,
+) -> Any:
+    """Compress messages, prepend legend to system prompt, call Claude API.
+
+    Lazy-imports the ``anthropic`` SDK.  All extra ``**kwargs`` are forwarded
+    to ``client.messages.create()``.
+
+    Returns the raw API response.
+    """
+    import anthropic  # noqa: lazy import
+
+    compressor = ClaudeCompressor(codebook)
+    compressed = compressor.compress_messages(messages)
+    legend = compressor.system_prompt_prefix()
+    instruction = (
+        "When responding with structured data (JSON/dicts), "
+        "use the [EL] short codes defined above for all keys and values that have mappings. "
+        "Pass through any keys or values not in the legend unchanged."
+    )
+    full_system = f"{legend}\n{instruction}\n\n{system}" if system else f"{legend}\n{instruction}"
+
+    client = anthropic.Anthropic()
+    return client.messages.create(
+        messages=compressed,
+        system=full_system,
+        **kwargs,
+    )