emergent-translator 1.1.0__py3-none-any.whl

emergent_translator/batch_encoder.py

@@ -0,0 +1,555 @@
+#!/usr/bin/env python3
+"""
+Batch Endpoint for Emergent Language
+
+Encodes multiple messages in a single request for enterprise use cases.
+
+Benefits:
+- Amortize HTTP overhead across many messages
+- Better compression (larger data = better zlib ratios)
+- Reduced connection churn
+- Atomic batch processing
+
+Usage:
+    from batch_encoder import BatchEncoder
+
+    encoder = BatchEncoder()
+    result = encoder.encode_batch([msg1, msg2, msg3, ...])
+
+    # result.compressed_bytes - the batch payload
+    # result.compression_ratio - overall compression
+    # result.messages_encoded - count
+"""
+
+import json
+import zlib
+import struct
+import time
+from dataclasses import dataclass
+from typing import List, Dict, Any, Tuple
+
+# ═══════════════════════════════════════════════════════════════════════════════
+# Emergent Symbol Encoder v2 (same as before)
+# ═══════════════════════════════════════════════════════════════════════════════
+
+COMMON_KEYS = {
+    "task": 0x01, "type": 0x02, "data": 0x03, "id": 0x04,
+    "agent": 0x05, "agent_id": 0x06, "task_type": 0x07,
+    "params": 0x08, "parameters": 0x08, "context": 0x09,
+    "priority": 0x0A, "status": 0x0B, "result": 0x0C,
+    "error": 0x0D, "message": 0x0E, "timestamp": 0x0F,
+    "action": 0x10, "target": 0x11, "source": 0x12, "value": 0x13,
+    "name": 0x14, "depth": 0x15, "level": 0x16, "mode": 0x17,
+    "config": 0x18, "settings": 0x19, "options": 0x1A,
+    "request": 0x1B, "response": 0x1C, "callback": 0x1D,
+    "coordination": 0x1E, "agents": 0x1F,
+    "version": 0x20, "v": 0x20, "capabilities": 0x21, "role": 0x22,
+    "task_id": 0x30, "request_id": 0x40, "initiator": 0x41,
+    "participants": 0x42, "task_distribution": 0x43, "consensus": 0x44,
+    "metrics": 0x50, "summary": 0x53, "confidence": 0x58,
+    "recommendations": 0x56, "timeframe": 0x59, "market": 0x60,
+    "sentiment": 0x68, "volatility": 0x69, "risk": 0x6A,
+    "order": 0x71, "symbol": 0x61, "side": 0x74, "quantity": 0x73, "limit": 0x75,
+}
+
+COMMON_VALUES = {
+    "analyze": 0x01, "optimize": 0x02, "execute": 0x03, "query": 0x04,
+    "high": 0x10, "medium": 0x11, "low": 0x12, "comprehensive": 0x18,
+    "pending": 0x20, "running": 0x21, "complete": 0x22, "completed": 0x22,
+    "failed": 0x23, "success": 0x26, "analysis": 0x30, "optimization": 0x31,
+    "execution": 0x32, "bullish": 0x50, "bearish": 0x51, "volatile": 0x53,
+    "buy": 0x56, "sell": 0x57, "hold": 0x58, "market": 0x60, "data": 0x61,
+    "trends": 0x62, "moderate": 0x66, "24h": 0x72, "quorum": 0x44,
+}
+
+# Reverse lookups for decoding
+COMMON_KEYS_REV = {v: k for k, v in COMMON_KEYS.items()}
+COMMON_VALUES_REV = {v: k for k, v in COMMON_VALUES.items()}
+
+
+def encode_dict(d: Dict, depth: int = 0, codebook=None) -> bytes:
+    if depth > 10 or not d:
+        return b'\x60'
+    keys_map = codebook.keys if codebook is not None else COMMON_KEYS
+    parts = [bytes([0x61, min(len(d), 255)])]
+    for k, v in list(d.items())[:255]:
+        kl = k.lower()
+        if kl in keys_map:
+            parts.append(bytes([0x80 | keys_map[kl]]))
+        else:
+            kb = k.encode()[:63]
+            parts.append(bytes([len(kb)]) + kb)
+        parts.append(encode_value(v, depth + 1, codebook=codebook))
+    return b''.join(parts)
+
+
+def encode_value(v: Any, depth: int = 0, codebook=None) -> bytes:
+    if v is None:
+        return b'\x00'
+    if isinstance(v, bool):
+        return bytes([0x01 if v else 0x02])
+    if isinstance(v, int):
+        if 0 <= v <= 127:
+            return bytes([0x10, v])
+        if -128 <= v <= 127:
+            return bytes([0x11, v & 0xFF])
+        if 0 <= v <= 65535:
+            return bytes([0x12]) + struct.pack('>H', v)
+        return bytes([0x13]) + struct.pack('>i', v)
+    if isinstance(v, float):
+        if v == int(v) and 0 <= v <= 65535:
+            return encode_value(int(v), depth, codebook=codebook)
+        return bytes([0x18]) + struct.pack('>f', v)
+    if isinstance(v, str):
+        vl = v.lower()
+        values_map = codebook.values if codebook is not None else COMMON_VALUES
+        if vl in values_map:
+            return bytes([0x20, values_map[vl]])
+        vb = v.encode()[:255]
+        if len(vb) < 16:
+            return bytes([0x30 | len(vb)]) + vb
+        return bytes([0x40, len(vb)]) + vb
+    if isinstance(v, list):
+        if not v:
+            return b'\x50'
+        parts = [bytes([0x51, min(len(v), 255)])]
+        for item in v[:255]:
+            parts.append(encode_value(item, depth + 1, codebook=codebook))
+        return b''.join(parts)
+    if isinstance(v, dict):
+        return encode_dict(v, depth + 1, codebook=codebook)
+    return encode_value(str(v), depth, codebook=codebook)
+
+
+def encode_single(data: Dict) -> Tuple[bytes, int, int]:
+    """Encode single message. Returns (bytes, original_size, compressed_size)."""
+    original = json.dumps(data, separators=(',', ':'))
+    original_size = len(original.encode())
+    raw = encode_dict(data)
+    compressed = zlib.compress(raw, 9)
+    if len(compressed) < len(raw) - 4:
+        payload = b'\x01' + compressed
+    else:
+        payload = b'\x00' + raw
+    final = b'\xE7\x02' + payload
+    return final, original_size, len(final)
+
+
+def decode_value(data: bytes, offset: int = 0, codebook=None) -> Tuple[Any, int]:
+    """Decode a single value from bytes. Returns (value, bytes_consumed)."""
+    marker = data[offset]
+    values_rev_map = codebook.values_rev if codebook is not None else COMMON_VALUES_REV
+
+    if marker == 0x00:
+        return None, 1
+    if marker == 0x01:
+        return True, 1
+    if marker == 0x02:
+        return False, 1
+
+    # Integers
+    if marker == 0x10:
+        return data[offset + 1], 2
+    if marker == 0x11:
+        v = data[offset + 1]
+        if v > 127:
+            v -= 256
+        return v, 2
+    if marker == 0x12:
+        return struct.unpack('>H', data[offset + 1:offset + 3])[0], 3
+    if marker == 0x13:
+        return struct.unpack('>i', data[offset + 1:offset + 5])[0], 5
+
+    # Float
+    if marker == 0x18:
+        return struct.unpack('>f', data[offset + 1:offset + 5])[0], 5
+
+    # Common value
+    if marker == 0x20:
+        vid = data[offset + 1]
+        return values_rev_map.get(vid, f"<unknown_value:{vid:#x}>"), 2
+
+    # Short string (0x30-0x3F)
+    if 0x30 <= marker <= 0x3F:
+        slen = marker & 0x0F
+        return data[offset + 1:offset + 1 + slen].decode('utf-8', errors='replace'), 1 + slen
+
+    # Medium string
+    if marker == 0x40:
+        slen = data[offset + 1]
+        return data[offset + 2:offset + 2 + slen].decode('utf-8', errors='replace'), 2 + slen
+
+    # Empty list
+    if marker == 0x50:
+        return [], 1
+
+    # List with count
+    if marker == 0x51:
+        count = data[offset + 1]
+        pos = 2
+        items = []
+        for _ in range(count):
+            val, consumed = decode_value(data, offset + pos, codebook=codebook)
+            items.append(val)
+            pos += consumed
+        return items, pos
+
+    # Dict
+    if marker == 0x60:
+        return {}, 1
+    if marker == 0x61:
+        val, consumed = decode_dict(data, offset, codebook=codebook)
+        return val, consumed
+
+    raise ValueError(f"Unknown marker {marker:#x} at offset {offset}")
+
+
+def decode_dict(data: bytes, offset: int = 0, codebook=None) -> Tuple[Dict, int]:
+    """Decode a dictionary from bytes. Returns (dict, bytes_consumed)."""
+    marker = data[offset]
+    if marker == 0x60:
+        return {}, 1
+
+    if marker != 0x61:
+        raise ValueError(f"Expected dict marker 0x60/0x61, got {marker:#x} at offset {offset}")
+
+    keys_rev_map = codebook.keys_rev if codebook is not None else COMMON_KEYS_REV
+
+    count = data[offset + 1]
+    pos = 2
+    result = {}
+
+    for _ in range(count):
+        # Decode key
+        kb = data[offset + pos]
+        if kb & 0x80:
+            key_id = kb & 0x7F
+            key = keys_rev_map.get(key_id, f"<unknown_key:{key_id:#x}>")
+            pos += 1
+        else:
+            klen = kb
+            key = data[offset + pos + 1:offset + pos + 1 + klen].decode('utf-8', errors='replace')
+            pos += 1 + klen
+
+        # Decode value
+        val, consumed = decode_value(data, offset + pos, codebook=codebook)
+        result[key] = val
+        pos += consumed
+
+    return result, pos
+
+
+# ═══════════════════════════════════════════════════════════════════════════════
+# Batch Encoder
+# ═══════════════════════════════════════════════════════════════════════════════
+
+@dataclass
+class BatchResult:
+    """Result of batch encoding."""
+    messages_encoded: int
+    original_bytes: int
+    compressed_bytes: int
+    compression_ratio: float
+    bandwidth_saved_pct: float
+    payload: bytes
+    encode_time_ms: float
+
+    # Comparison with individual encoding
+    individual_bytes: int = 0
+    batch_advantage_pct: float = 0.0
+
+
+class BatchEncoder:
+    """
+    Batch encoder for multiple messages.
+
+    Format (v1/v2):
+        MAGIC (2 bytes): 0xE7 0xB0 (θ batch)
+        VERSION (1 byte): 0x01
+        COUNT (2 bytes): number of messages (big-endian)
+        COMPRESSED (1 byte): 0x00 = raw, 0x01 = zlib
+        PAYLOAD: concatenated encoded messages (each prefixed with 2-byte length)
+        CHECKSUM (4 bytes): CRC32
+
+    Format (v3 — adaptive codebook):
+        MAGIC (2) + VERSION=0x03 (1) + COUNT (2) + FLAGS (1)
+        + CB_VERSION (2) + CB_LEN (2) + EMBEDDED_CB + PAYLOAD + CRC (4)
+    """
+
+    MAGIC = b'\xE7\xB0'  # θ batch
+    VERSION = 0x01
+    VERSION_ADAPTIVE = 0x03
+
+    def __init__(self, codebook=None):
+        self._codebook = codebook
+
+    def encode_batch(self, messages: List[Dict], embed_codebook: bool = True) -> BatchResult:
+        """Encode multiple messages into a single batch payload.
+
+        Args:
+            messages: List of dicts to encode.
+            embed_codebook: If True (default), embed the codebook in the v3
+                header so payloads are self-contained.  Set to False when both
+                sides already share the codebook to avoid per-batch overhead.
+        """
+        start = time.perf_counter()
+
+        # Auto-observe messages for codebook learning
+        if self._codebook is not None:
+            self._codebook.observe(messages)
+
+        # Capture active codebook snapshot
+        active_cb = self._codebook.get_active() if self._codebook is not None else None
+
+        # Calculate original size (JSON)
+        original_json = [json.dumps(m, separators=(',', ':')) for m in messages]
+        original_bytes = sum(len(j.encode()) for j in original_json)
+
+        # Encode each message individually (for comparison)
+        individual_encoded = []
+        for msg in messages:
+            encoded, _, size = encode_single(msg)
+            individual_encoded.append(encoded)
+        individual_bytes = sum(len(e) for e in individual_encoded)
+
+        # Batch encode - concatenate raw encodings, then compress together
+        raw_messages = []
+        for msg in messages:
+            raw = encode_dict(msg, codebook=active_cb)
+            # Prefix each with 2-byte length
+            raw_messages.append(struct.pack('>H', len(raw)) + raw)
+
+        combined_raw = b''.join(raw_messages)
+
+        # Compress the combined payload
+        compressed = zlib.compress(combined_raw, 9)
+
+        # Use compressed if smaller
+        if len(compressed) < len(combined_raw):
+            payload_data = compressed
+            compression_flag = 0x01
+        else:
+            payload_data = combined_raw
+            compression_flag = 0x00
+
+        # Build batch header
+        if active_cb is not None:
+            # V3 header — optionally embed the codebook
+            if embed_codebook:
+                flags = compression_flag | 0x02  # bit 1 = codebook embedded
+                cb_serialized = active_cb.serialize()
+            else:
+                flags = compression_flag  # bit 1 clear = no embedded codebook
+                cb_serialized = b''
+            header = (
+                self.MAGIC +
+                bytes([self.VERSION_ADAPTIVE]) +
+                struct.pack('>H', len(messages)) +
+                bytes([flags]) +
+                struct.pack('>H', active_cb.version) +
+                struct.pack('>H', len(cb_serialized)) +
+                cb_serialized
+            )
+        else:
+            header = (
+                self.MAGIC +
+                bytes([self.VERSION]) +
+                struct.pack('>H', len(messages)) +
+                bytes([compression_flag])
+            )
+
+        # Final payload
+        payload = header + payload_data
+
+        # Add checksum
+        checksum = zlib.crc32(payload) & 0xFFFFFFFF
+        final_payload = payload + struct.pack('>I', checksum)
+
+        encode_time = (time.perf_counter() - start) * 1000
+
+        compressed_bytes = len(final_payload)
+        compression_ratio = compressed_bytes / original_bytes if original_bytes > 0 else 1.0
+        bandwidth_saved = (1 - compression_ratio) * 100
+
+        # Batch advantage over individual encoding
+        batch_advantage = (1 - compressed_bytes / individual_bytes) * 100 if individual_bytes > 0 else 0
+
+        return BatchResult(
+            messages_encoded=len(messages),
+            original_bytes=original_bytes,
+            compressed_bytes=compressed_bytes,
+            compression_ratio=compression_ratio,
+            bandwidth_saved_pct=bandwidth_saved,
+            payload=final_payload,
+            encode_time_ms=encode_time,
+            individual_bytes=individual_bytes,
+            batch_advantage_pct=batch_advantage
+        )
+
+    def decode_batch_header(self, payload: bytes) -> dict:
+        """Decode batch header (for inspection)."""
+        if len(payload) < 10:
+            return {"error": "payload too short"}
+
+        magic = payload[:2]
+        if magic != self.MAGIC:
+            return {"error": f"invalid magic: {magic.hex()}"}
+
+        version = payload[2]
+        count = struct.unpack('>H', payload[3:5])[0]
+        flags = payload[5]
+
+        info = {
+            "magic": magic.hex(),
+            "version": version,
+            "message_count": count,
+            "compressed": bool(flags & 0x01),
+            "payload_size": len(payload),
+        }
+
+        if version == 0x03:
+            info["codebook_embedded"] = bool(flags & 0x02)
+            info["codebook_version"] = struct.unpack('>H', payload[6:8])[0]
+
+        return info
+
+    def decode_batch(self, payload: bytes, codebook=None) -> List[Dict]:
+        """Decode a batch payload back into a list of message dicts.
+
+        Args:
+            payload: The binary batch payload.
+            codebook: Optional external CodebookVersion for v3 payloads
+                encoded with ``embed_codebook=False``.  Ignored when the
+                payload already contains an embedded codebook.
+        """
+        if len(payload) < 10:
+            raise ValueError("Payload too short")
+
+        # Verify checksum (last 4 bytes)
+        stored_crc = struct.unpack('>I', payload[-4:])[0]
+        computed_crc = zlib.crc32(payload[:-4]) & 0xFFFFFFFF
+        if stored_crc != computed_crc:
+            raise ValueError(f"CRC32 mismatch: stored={stored_crc:#x}, computed={computed_crc:#x}")
+
+        # Parse header
+        magic = payload[:2]
+        if magic != self.MAGIC:
+            raise ValueError(f"Invalid magic: {magic.hex()}")
+
+        version = payload[2]
+        if version not in (0x01, 0x02, 0x03):
+            raise ValueError(f"Unsupported version: {version:#x}")
+
+        count = struct.unpack('>H', payload[3:5])[0]
+        flags = payload[5]
+
+        cb = codebook  # external codebook (may be None)
+        if version == 0x03:
+            # V3: parse FLAGS, CB_VERSION, embedded codebook
+            compression_flag = flags & 0x01
+            cb_embedded = bool(flags & 0x02)
+            cb_version_num = struct.unpack('>H', payload[6:8])[0]
+            cb_len = struct.unpack('>H', payload[8:10])[0]
+            data_offset = 10
+
+            if cb_embedded:
+                from .adaptive_codebook import CodebookVersion
+                cb, _ = CodebookVersion.deserialize(payload, data_offset)
+                data_offset += cb_len
+
+            raw_data = payload[data_offset:-4]
+        else:
+            # V1/V2: simple header
+            compression_flag = flags
+            raw_data = payload[6:-4]
+
+        # Decompress if needed
+        if compression_flag & 0x01:
+            raw_data = zlib.decompress(raw_data)
+
+        # Split messages by 2-byte length prefixes
+        messages = []
+        pos = 0
+        for _ in range(count):
+            msg_len = struct.unpack('>H', raw_data[pos:pos + 2])[0]
+            pos += 2
+            msg_bytes = raw_data[pos:pos + msg_len]
+            msg, _ = decode_dict(msg_bytes, 0, codebook=cb)
+            messages.append(msg)
+            pos += msg_len
+
+        return messages
+
+
+# ═══════════════════════════════════════════════════════════════════════════════
+# Demo
+# ═══════════════════════════════════════════════════════════════════════════════
+
+SAMPLE_MESSAGES = [
+    {"task": "analyze", "data": "market trends", "priority": "high"},
+    {"agent_id": "agent_001", "task_type": "analysis", "status": "pending"},
+    {"agent_id": "agent_002", "task_type": "execution", "status": "running"},
+    {"coordination": {"request_id": "coord_001", "participants": ["a1", "a2", "a3"]}},
+    {"result": {"summary": "bullish", "confidence": 0.85, "recommendations": ["buy"]}},
+    {"task_id": "task_001", "status": "complete", "metrics": {"latency": 45, "success": True}},
+    {"agent": {"id": "trader", "version": "2.0"}, "context": {"market": "volatile"}},
+    {"order": {"symbol": "BTC", "side": "buy", "quantity": 100, "status": "pending"}},
+    {"task_type": "optimization", "priority": "high", "context": {"risk": "low"}},
+    {"status": "complete", "result": {"value": 42, "confidence": 0.95}},
+]
+
+
+def main():
+    import random
+
+    print("\n" + "═" * 70)
+    print("  🎯 BATCH ENCODER DEMO")
+    print("═" * 70 + "\n")
+
+    encoder = BatchEncoder()
+
+    # Test different batch sizes
+    for batch_size in [10, 50, 100, 500, 1000]:
+        # Generate batch
+        messages = [random.choice(SAMPLE_MESSAGES) for _ in range(batch_size)]
+
+        result = encoder.encode_batch(messages)
+
+        print(f"  Batch size: {batch_size:,} messages")
+        print(f"    Original (JSON):     {result.original_bytes:,} bytes")
+        print(f"    Individual encoded:  {result.individual_bytes:,} bytes")
+        print(f"    Batch encoded:       {result.compressed_bytes:,} bytes")
+        print(f"    Compression:         {result.bandwidth_saved_pct:.1f}% saved")
+        print(f"    Batch advantage:     {result.batch_advantage_pct:.1f}% smaller than individual")
+        print(f"    Encode time:         {result.encode_time_ms:.2f}ms")
+        print()
+
+    print("═" * 70)
+    print("  📊 BATCH vs INDIVIDUAL COMPARISON")
+    print("═" * 70 + "\n")
+
+    # Detailed comparison for 100 messages
+    messages = [random.choice(SAMPLE_MESSAGES) for _ in range(100)]
+    result = encoder.encode_batch(messages)
+
+    print(f"  100 agent messages:")
+    print(f"    ┌─────────────────────────────────────────┐")
+    print(f"    │ Method          │ Size      │ Savings  │")
+    print(f"    ├─────────────────────────────────────────┤")
+    print(f"    │ Raw JSON        │ {result.original_bytes:>6,} B  │    -     │")
+    print(f"    │ Individual θ    │ {result.individual_bytes:>6,} B  │ {(1-result.individual_bytes/result.original_bytes)*100:>5.1f}%  │")
+    print(f"    │ Batch θ         │ {result.compressed_bytes:>6,} B  │ {result.bandwidth_saved_pct:>5.1f}%  │")
+    print(f"    └─────────────────────────────────────────┘")
+    print()
+    print(f"  Batch is {result.batch_advantage_pct:.1f}% smaller than individual encoding!")
+    print(f"  Total savings vs JSON: {result.bandwidth_saved_pct:.1f}%")
+    print()
+
+    # Show header info
+    header = encoder.decode_batch_header(result.payload)
+    print(f"  Batch header: {header}")
+    print()
+
+
+if __name__ == "__main__":
+    main()