emergent-translator 1.1.0__py3-none-any.whl

emergent_translator/__init__.py

@@ -0,0 +1,126 @@
+"""
+Emergent Language Translator
+
+A high-performance API for translating between traditional AI communication formats
+and emergent language symbols with 60x compression efficiency.
+
+Example:
+    >>> from emergent_translator import BatchEncoder
+    >>> encoder = BatchEncoder()
+    >>> result = encoder.encode_batch([{"role": "user", "content": "hello"}])
+"""
+
+# --- Always-available imports (no heavy deps) ---
+from .batch_encoder import (
+    BatchEncoder,
+    BatchResult,
+    COMMON_KEYS,
+    COMMON_VALUES,
+)
+
+from .adaptive_codebook import (
+    AdaptiveCodebook,
+    CodebookVersion,
+    FrequencyTracker,
+)
+
+from .format_handlers import (
+    detect_format,
+    get_handler,
+    is_binary_format,
+)
+
+from .emergent_symbols import EmergentSymbolEncoder
+
+__version__ = "1.1.0"
+__author__ = "Emergent Language Team"
+__email__ = "hello@emergentlanguage.ai"
+__description__ = "60x compression efficiency for AI communication"
+__url__ = "https://github.com/maco144/emergent-language"
+
+__all__ = [
+    # Batch encoder
+    "BatchEncoder",
+    "BatchResult",
+    "COMMON_KEYS",
+    "COMMON_VALUES",
+
+    # Adaptive codebook
+    "AdaptiveCodebook",
+    "CodebookVersion",
+    "FrequencyTracker",
+
+    # Format handlers
+    "detect_format",
+    "get_handler",
+    "is_binary_format",
+
+    # Symbols
+    "EmergentSymbolEncoder",
+
+    # GPU (lazy-loaded)
+    "GPUBatchEncoder",
+
+    # SDK / server components (lazy-loaded, need extra deps)
+    "TranslatorSDK",
+    "EmergentTranslatorClient",
+    "SyncEmergentTranslatorClient",
+    "EmergentLanguageTranslator",
+    "ChunkCoordinator",
+    "ChunkCollector",
+
+    # Metadata
+    "__version__",
+    "__author__",
+    "__email__",
+    "__description__",
+    "__url__",
+]
+
+# --- Lazy imports for modules with heavy/optional dependencies ---
+_LAZY_IMPORTS = {
+    # GPU encoder (cupy detection prints at import time)
+    "GPUBatchEncoder": (".gpu_batch_encoder", "GPUBatchEncoder"),
+    # SDK / client
+    "TranslatorSDK": (".client_sdk", "TranslatorSDK"),
+    "EmergentTranslatorClient": (".client_sdk", "EmergentTranslatorClient"),
+    "SyncEmergentTranslatorClient": (".client_sdk", "SyncEmergentTranslatorClient"),
+    # Core (needs eudaimonia)
+    "EmergentLanguageTranslator": (".core", "EmergentLanguageTranslator"),
+    "TranslationFormat": (".core", "TranslationFormat"),
+    "TranslationDirection": (".core", "TranslationDirection"),
+    "TranslationResult": (".core", "TranslationResult"),
+    "TranslationStats": (".core", "TranslationStats"),
+    # Distributed processing
+    "ChunkCoordinator": (".chunk_coordinator", "ChunkCoordinator"),
+    "DistributedJob": (".chunk_coordinator", "DistributedJob"),
+    "Chunk": (".chunk_coordinator", "Chunk"),
+    "ChunkStatus": (".chunk_coordinator", "ChunkStatus"),
+    "JobStatus": (".chunk_coordinator", "JobStatus"),
+    "distributed_process": (".chunk_coordinator", "distributed_process"),
+    # Chunk collector
+    "ChunkCollector": (".chunk_collector", "ChunkCollector"),
+    "create_collector_app": (".chunk_collector", "create_collector_app"),
+    "CollectorStats": (".chunk_collector", "CollectorStats"),
+    # Claude compression
+    "TextCodebook": (".claude_compression", "TextCodebook"),
+    "ClaudeCompressor": (".claude_compression", "ClaudeCompressor"),
+    "text_compress": (".claude_compression", "compress"),
+    "text_decompress": (".claude_compression", "decompress"),
+    "estimate_tokens": (".claude_compression", "estimate_tokens"),
+    # Code skeleton
+    "skeletonize": (".code_skeleton", "skeletonize"),
+    "skeletonize_file": (".code_skeleton", "skeletonize_file"),
+    "skeletonize_dir": (".code_skeleton", "skeletonize_dir"),
+    "SkeletonResult": (".code_skeleton", "SkeletonResult"),
+    "CodeSkeleton": (".code_skeleton", "CodeSkeleton"),
+}
+
+
+def __getattr__(name):
+    if name in _LAZY_IMPORTS:
+        module_path, attr = _LAZY_IMPORTS[name]
+        import importlib
+        mod = importlib.import_module(module_path, __name__)
+        return getattr(mod, attr)
+    raise AttributeError(f"module {__name__!r} has no attribute {name!r}")

emergent_translator/adaptive_codebook.py

@@ -0,0 +1,342 @@
+#!/usr/bin/env python3
+"""
+Adaptive Learned Codebook for Compression
+
+Replaces hardcoded COMMON_KEYS/COMMON_VALUES with a codebook that learns
+from observed traffic. Pure frequency-based, no governance.
+
+Architecture:
+    AdaptiveCodebook
+    ├── FrequencyTracker       # Counts key/value occurrences (thread-safe)
+    ├── CodebookVersion        # Immutable snapshot (frozen dataclass)
+    ├── versions: Dict[int, CodebookVersion]
+    ├── active: CodebookVersion
+    ├── rebuild(min_freq)      # Create new version from accumulated data
+    └── save()/load()          # JSON persistence
+"""
+
+import json
+import struct
+import threading
+import time
+from dataclasses import dataclass, field
+from typing import Any, Dict, List, Optional, Tuple
+
+# Import static dictionaries for baseline codebook (version 0)
+from .batch_encoder import COMMON_KEYS, COMMON_VALUES
+
+
+# =============================================================================
+# FrequencyTracker
+# =============================================================================
+
+class FrequencyTracker:
+    """Thread-safe frequency counter for keys and string values."""
+
+    def __init__(self):
+        self._lock = threading.Lock()
+        self._key_counts: Dict[str, int] = {}
+        self._value_counts: Dict[str, int] = {}
+        self._total_messages: int = 0
+
+    def observe(self, msg: Dict[str, Any]) -> None:
+        """Record keys and string values from a single message."""
+        with self._lock:
+            self._total_messages += 1
+            self._extract(msg, depth=0)
+
+    def observe_batch(self, messages: List[Dict[str, Any]]) -> None:
+        """Record keys and string values from multiple messages."""
+        with self._lock:
+            for msg in messages:
+                self._total_messages += 1
+                self._extract(msg, depth=0)
+
+    def _extract(self, obj: Any, depth: int) -> None:
+        """Recursively pull keys and short string values, lowercased.
+
+        Must be called while holding self._lock.
+        """
+        if depth > 10:
+            return
+        if isinstance(obj, dict):
+            for k, v in obj.items():
+                kl = k.lower()
+                self._key_counts[kl] = self._key_counts.get(kl, 0) + 1
+                self._extract(v, depth + 1)
+        elif isinstance(obj, list):
+            for item in obj:
+                self._extract(item, depth + 1)
+        elif isinstance(obj, str):
+            vl = obj.lower()
+            if len(vl) <= 64:
+                self._value_counts[vl] = self._value_counts.get(vl, 0) + 1
+
+    def get_top_keys(self, n: int) -> List[Tuple[str, int]]:
+        """Return top-n keys ranked by frequency."""
+        with self._lock:
+            items = sorted(self._key_counts.items(), key=lambda x: -x[1])
+            return items[:n]
+
+    def get_top_values(self, n: int) -> List[Tuple[str, int]]:
+        """Return top-n values ranked by frequency."""
+        with self._lock:
+            items = sorted(self._value_counts.items(), key=lambda x: -x[1])
+            return items[:n]
+
+    @property
+    def total_messages(self) -> int:
+        with self._lock:
+            return self._total_messages
+
+    def reset(self) -> None:
+        """Clear all counts."""
+        with self._lock:
+            self._key_counts.clear()
+            self._value_counts.clear()
+            self._total_messages = 0
+
+
+# =============================================================================
+# CodebookVersion
+# =============================================================================
+
+@dataclass(frozen=True)
+class CodebookVersion:
+    """Immutable snapshot of a codebook mapping."""
+    version: int
+    keys: Dict[str, int]        # str → byte_id (0x01-0x7F)
+    values: Dict[str, int]      # str → byte_id (0x01-0x7F)
+    keys_rev: Dict[int, str]    # byte_id → str
+    values_rev: Dict[int, str]  # byte_id → str
+    trained_on: int = 0         # number of messages used to train
+    created_at: float = 0.0     # timestamp
+
+    def serialize(self) -> bytes:
+        """Compact binary serialization for batch headers.
+
+        Format:
+            KEY_COUNT(1) + [ID(1) + LEN(1) + BYTES]...
+            + VAL_COUNT(1) + [ID(1) + LEN(1) + BYTES]...
+        """
+        parts = []
+        # Keys
+        key_items = sorted(self.keys.items(), key=lambda x: x[1])
+        parts.append(bytes([len(key_items)]))
+        for name, byte_id in key_items:
+            name_bytes = name.encode('utf-8')[:255]
+            parts.append(bytes([byte_id, len(name_bytes)]) + name_bytes)
+        # Values
+        val_items = sorted(self.values.items(), key=lambda x: x[1])
+        parts.append(bytes([len(val_items)]))
+        for name, byte_id in val_items:
+            name_bytes = name.encode('utf-8')[:255]
+            parts.append(bytes([byte_id, len(name_bytes)]) + name_bytes)
+        return b''.join(parts)
+
+    @staticmethod
+    def deserialize(data: bytes, offset: int = 0) -> Tuple['CodebookVersion', int]:
+        """Reconstruct a CodebookVersion from bytes.
+
+        Returns (CodebookVersion, bytes_consumed).
+        """
+        pos = offset
+        # Keys
+        key_count = data[pos]; pos += 1
+        keys = {}
+        for _ in range(key_count):
+            byte_id = data[pos]; pos += 1
+            name_len = data[pos]; pos += 1
+            name = data[pos:pos + name_len].decode('utf-8')
+            pos += name_len
+            keys[name] = byte_id
+        # Values
+        val_count = data[pos]; pos += 1
+        values = {}
+        for _ in range(val_count):
+            byte_id = data[pos]; pos += 1
+            name_len = data[pos]; pos += 1
+            name = data[pos:pos + name_len].decode('utf-8')
+            pos += name_len
+            values[name] = byte_id
+
+        keys_rev = {v: k for k, v in keys.items()}
+        values_rev = {v: k for k, v in values.items()}
+
+        cb = CodebookVersion(
+            version=0,  # Will be set by caller if needed
+            keys=keys,
+            values=values,
+            keys_rev=keys_rev,
+            values_rev=values_rev,
+        )
+        return cb, pos - offset
+
+    def to_dict(self) -> Dict[str, Any]:
+        """JSON-serializable representation."""
+        return {
+            "version": self.version,
+            "keys": self.keys,
+            "values": self.values,
+            "trained_on": self.trained_on,
+            "created_at": self.created_at,
+        }
+
+    @staticmethod
+    def from_dict(d: Dict[str, Any]) -> 'CodebookVersion':
+        """Reconstruct from JSON dict."""
+        keys = {k: v for k, v in d["keys"].items()}
+        values = {k: v for k, v in d["values"].items()}
+        keys_rev = {v: k for k, v in keys.items()}
+        values_rev = {v: k for k, v in values.items()}
+        return CodebookVersion(
+            version=d["version"],
+            keys=keys,
+            values=values,
+            keys_rev=keys_rev,
+            values_rev=values_rev,
+            trained_on=d.get("trained_on", 0),
+            created_at=d.get("created_at", 0.0),
+        )
+
+
+# =============================================================================
+# Baseline codebook (version 0) from static dictionaries
+# =============================================================================
+
+def make_baseline_codebook() -> CodebookVersion:
+    """Create version 0 codebook from the static COMMON_KEYS/COMMON_VALUES."""
+    keys_rev = {v: k for k, v in COMMON_KEYS.items()}
+    values_rev = {v: k for k, v in COMMON_VALUES.items()}
+    return CodebookVersion(
+        version=0,
+        keys=dict(COMMON_KEYS),
+        values=dict(COMMON_VALUES),
+        keys_rev=keys_rev,
+        values_rev=values_rev,
+        trained_on=0,
+        created_at=time.time(),
+    )
+
+
+# =============================================================================
+# AdaptiveCodebook
+# =============================================================================
+
+class AdaptiveCodebook:
+    """Manages learned codebook versions with frequency tracking."""
+
+    def __init__(self, persist_path: Optional[str] = None):
+        self._persist_path = persist_path
+        self._tracker = FrequencyTracker()
+        self._versions: Dict[int, CodebookVersion] = {}
+        self._active: CodebookVersion = make_baseline_codebook()
+        self._versions[0] = self._active
+        self._next_version = 1
+
+    @property
+    def tracker(self) -> FrequencyTracker:
+        return self._tracker
+
+    def observe(self, messages: List[Dict[str, Any]]) -> None:
+        """Feed messages to the frequency tracker."""
+        self._tracker.observe_batch(messages)
+
+    def rebuild(
+        self,
+        min_frequency: int = 10,
+        max_keys: int = 127,
+        max_values: int = 127,
+    ) -> CodebookVersion:
+        """Build a new codebook version from accumulated frequency data.
+
+        Keys/values with count >= min_frequency are assigned IDs starting at 0x01.
+        Returns the new CodebookVersion (also set as active).
+        """
+        top_keys = self._tracker.get_top_keys(max_keys)
+        top_values = self._tracker.get_top_values(max_values)
+
+        # Filter by minimum frequency
+        top_keys = [(k, c) for k, c in top_keys if c >= min_frequency]
+        top_values = [(v, c) for v, c in top_values if c >= min_frequency]
+
+        # Assign IDs starting at 0x01
+        keys = {}
+        for i, (k, _) in enumerate(top_keys):
+            keys[k] = i + 1  # 0x01, 0x02, ...
+
+        values = {}
+        for i, (v, _) in enumerate(top_values):
+            values[v] = i + 1
+
+        keys_rev = {v: k for k, v in keys.items()}
+        values_rev = {v: k for k, v in values.items()}
+
+        version_num = self._next_version
+        self._next_version += 1
+
+        cb = CodebookVersion(
+            version=version_num,
+            keys=keys,
+            values=values,
+            keys_rev=keys_rev,
+            values_rev=values_rev,
+            trained_on=self._tracker.total_messages,
+            created_at=time.time(),
+        )
+
+        self._versions[version_num] = cb
+        self._active = cb
+        return cb
+
+    def get_active(self) -> CodebookVersion:
+        """Return the current active codebook version."""
+        return self._active
+
+    def get_version(self, v: int) -> Optional[CodebookVersion]:
+        """Return a specific codebook version, or None if not found."""
+        return self._versions.get(v)
+
+    def get_stats(self) -> Dict[str, Any]:
+        """Return statistics about the codebook."""
+        return {
+            "active_version": self._active.version,
+            "total_versions": len(self._versions),
+            "tracked_messages": self._tracker.total_messages,
+            "tracked_keys": len(self._tracker.get_top_keys(999999)),
+            "tracked_values": len(self._tracker.get_top_values(999999)),
+            "active_keys": len(self._active.keys),
+            "active_values": len(self._active.values),
+        }
+
+    def save(self) -> None:
+        """Persist codebook to JSON file."""
+        path = self._persist_path
+        if path is None:
+            raise ValueError("No persist_path configured")
+        data = {
+            "next_version": self._next_version,
+            "active_version": self._active.version,
+            "versions": {
+                str(v): cb.to_dict() for v, cb in self._versions.items()
+            },
+        }
+        with open(path, 'w') as f:
+            json.dump(data, f, indent=2)
+
+    def load(self) -> None:
+        """Load codebook from JSON file."""
+        path = self._persist_path
+        if path is None:
+            raise ValueError("No persist_path configured")
+        with open(path, 'r') as f:
+            data = json.load(f)
+
+        self._next_version = data["next_version"]
+        self._versions.clear()
+        for v_str, cb_dict in data["versions"].items():
+            cb = CodebookVersion.from_dict(cb_dict)
+            self._versions[cb.version] = cb
+
+        active_v = data["active_version"]
+        self._active = self._versions[active_v]