headroom-ai 0.2.13__py3-none-any.whl

headroom/telemetry/toin.py

@@ -0,0 +1,1579 @@
+"""Tool Output Intelligence Network (TOIN) - Cross-user learning for compression.
+
+TOIN aggregates anonymized compression patterns across all Headroom users to
+create a network effect: every user's compression decisions improve the
+recommendations for everyone.
+
+Key concepts:
+- ToolPattern: Aggregated intelligence about a tool type (by structure hash)
+- CompressionHint: Recommendations for how to compress a specific tool output
+- ToolIntelligenceNetwork: Central aggregator that learns from all users
+
+How it works:
+1. When SmartCrusher compresses data, it records the outcome via telemetry
+2. When LLM retrieves compressed data, TOIN tracks what was needed
+3. TOIN learns: "For tools with structure X, retrieval rate is high when
+   compressing field Y - preserve it"
+4. Next time: SmartCrusher asks TOIN for hints before compressing
+
+Privacy:
+- No actual data values are stored
+- Tool names are structure hashes
+- Field names are SHA256[:8] hashes
+- No user identifiers
+
+Network Effect:
+- More users → more compression events → better recommendations
+- Cross-user patterns reveal universal tool behaviors
+- Federated learning: aggregate patterns, not data
+
+Usage:
+    from headroom.telemetry.toin import get_toin
+
+    # Before compression, get recommendations
+    hint = get_toin().get_recommendation(tool_signature, query_context)
+
+    # Apply hint
+    if hint.skip_compression:
+        return original_data
+    config.preserve_fields = hint.preserve_fields
+    config.max_items = hint.max_items
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import logging
+import threading
+import time
+from collections.abc import Callable
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any, Literal
+
+from .models import FieldSemantics, ToolSignature
+
+logger = logging.getLogger(__name__)
+
+# LOW FIX #22: Define callback types for metrics/monitoring hooks
+# These allow users to plug in their own metrics collection (Prometheus, StatsD, etc.)
+MetricsCallback = Callable[[str, dict[str, Any]], None]  # (event_name, event_data) -> None
+
+
+@dataclass
+class ToolPattern:
+    """Aggregated intelligence about a tool type across all users.
+
+    This is the core TOIN data structure. It represents everything we've
+    learned about how to compress outputs from tools with a specific structure.
+    """
+
+    tool_signature_hash: str
+
+    # === Compression Statistics ===
+    total_compressions: int = 0
+    total_items_seen: int = 0
+    total_items_kept: int = 0
+    avg_compression_ratio: float = 0.0
+    avg_token_reduction: float = 0.0
+
+    # === Retrieval Statistics ===
+    total_retrievals: int = 0
+    full_retrievals: int = 0  # Retrieved everything
+    search_retrievals: int = 0  # Used search filter
+
+    @property
+    def retrieval_rate(self) -> float:
+        """Fraction of compressions that triggered retrieval."""
+        if self.total_compressions == 0:
+            return 0.0
+        return self.total_retrievals / self.total_compressions
+
+    @property
+    def full_retrieval_rate(self) -> float:
+        """Fraction of retrievals that were full (not search)."""
+        if self.total_retrievals == 0:
+            return 0.0
+        return self.full_retrievals / self.total_retrievals
+
+    # === Learned Patterns ===
+    # Fields that are frequently retrieved (should preserve)
+    commonly_retrieved_fields: list[str] = field(default_factory=list)
+    field_retrieval_frequency: dict[str, int] = field(default_factory=dict)
+
+    # Query patterns that trigger retrieval
+    common_query_patterns: list[str] = field(default_factory=list)
+    # MEDIUM FIX #10: Track query pattern frequency to keep most common, not just recent
+    query_pattern_frequency: dict[str, int] = field(default_factory=dict)
+
+    # Best compression strategy for this tool type
+    optimal_strategy: str = "default"
+    strategy_success_rates: dict[str, float] = field(default_factory=dict)
+
+    # === Learned Recommendations ===
+    optimal_max_items: int = 20
+    skip_compression_recommended: bool = False
+    preserve_fields: list[str] = field(default_factory=list)
+
+    # === Field-Level Semantics (TOIN Evolution) ===
+    # Learned semantic types for each field based on retrieval patterns
+    # This enables zero-latency signal detection without hardcoded patterns
+    field_semantics: dict[str, FieldSemantics] = field(default_factory=dict)
+
+    # === Confidence ===
+    sample_size: int = 0
+    user_count: int = 0  # Number of unique users (anonymized)
+    confidence: float = 0.0  # 0.0 = no data, 1.0 = high confidence
+    last_updated: float = 0.0
+
+    # === Instance Tracking (for user_count) ===
+    # Hashed instance IDs of users who have contributed to this pattern
+    # Limited to avoid unbounded growth (for serialization)
+    _seen_instance_hashes: list[str] = field(default_factory=list)
+    # FIX: Separate set for ALL seen instances to prevent double-counting
+    # CRITICAL FIX #1: Capped at MAX_SEEN_INSTANCES to prevent OOM with millions of users.
+    # When cap is reached, we rely on user_count for accurate counting and
+    # accept some potential double-counting for new users (negligible at scale).
+    _all_seen_instances: set[str] = field(default_factory=set)
+
+    # CRITICAL FIX: Track whether instance tracking was truncated during serialization
+    # If True, we know some users were lost and should be conservative about user_count
+    _tracking_truncated: bool = False
+
+    # CRITICAL FIX #1: Maximum entries in _all_seen_instances to prevent OOM
+    # This is a class constant, not a field (not serialized)
+    MAX_SEEN_INSTANCES: int = 10000
+
+    def to_dict(self) -> dict[str, Any]:
+        """Convert to dictionary for serialization."""
+        return {
+            "tool_signature_hash": self.tool_signature_hash,
+            "total_compressions": self.total_compressions,
+            "total_items_seen": self.total_items_seen,
+            "total_items_kept": self.total_items_kept,
+            "avg_compression_ratio": self.avg_compression_ratio,
+            "avg_token_reduction": self.avg_token_reduction,
+            "total_retrievals": self.total_retrievals,
+            "full_retrievals": self.full_retrievals,
+            "search_retrievals": self.search_retrievals,
+            "retrieval_rate": self.retrieval_rate,
+            "full_retrieval_rate": self.full_retrieval_rate,
+            "commonly_retrieved_fields": self.commonly_retrieved_fields,
+            "field_retrieval_frequency": self.field_retrieval_frequency,
+            "common_query_patterns": self.common_query_patterns,
+            "query_pattern_frequency": self.query_pattern_frequency,
+            "optimal_strategy": self.optimal_strategy,
+            "strategy_success_rates": self.strategy_success_rates,
+            "optimal_max_items": self.optimal_max_items,
+            "skip_compression_recommended": self.skip_compression_recommended,
+            "preserve_fields": self.preserve_fields,
+            # Field-level semantics (TOIN Evolution)
+            "field_semantics": {k: v.to_dict() for k, v in self.field_semantics.items()},
+            "sample_size": self.sample_size,
+            "user_count": self.user_count,
+            "confidence": self.confidence,
+            "last_updated": self.last_updated,
+            # Serialize instance hashes (limited to 100 for bounded storage)
+            "seen_instance_hashes": self._seen_instance_hashes[:100],
+            # CRITICAL FIX: Track if truncation occurred during serialization
+            # This tells from_dict() that some users were lost and prevents double-counting
+            "tracking_truncated": (
+                self._tracking_truncated
+                or self.user_count > len(self._seen_instance_hashes)
+                or len(self._all_seen_instances) > 100
+            ),
+        }
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any]) -> ToolPattern:
+        """Create from dictionary."""
+        # Filter to only valid fields
+        valid_fields = {
+            "tool_signature_hash",
+            "total_compressions",
+            "total_items_seen",
+            "total_items_kept",
+            "avg_compression_ratio",
+            "avg_token_reduction",
+            "total_retrievals",
+            "full_retrievals",
+            "search_retrievals",
+            "commonly_retrieved_fields",
+            "field_retrieval_frequency",
+            "common_query_patterns",
+            "query_pattern_frequency",
+            "optimal_strategy",
+            "strategy_success_rates",
+            "optimal_max_items",
+            "skip_compression_recommended",
+            "preserve_fields",
+            "sample_size",
+            "user_count",
+            "confidence",
+            "last_updated",
+        }
+        filtered = {k: v for k, v in data.items() if k in valid_fields}
+
+        # Handle seen_instance_hashes (serialized without underscore prefix)
+        seen_hashes = data.get("seen_instance_hashes", [])
+
+        pattern = cls(**filtered)
+        pattern._seen_instance_hashes = seen_hashes[:100]  # Limit on load
+
+        # CRITICAL FIX: Populate _all_seen_instances from loaded hashes
+        # This prevents double-counting after restart - without this, the same
+        # instances would be counted again because the lookup set was empty
+        pattern._all_seen_instances = set(pattern._seen_instance_hashes)
+
+        # CRITICAL FIX: Restore truncation flag to prevent double-counting
+        # If truncated, we know some users were lost in serialization
+        pattern._tracking_truncated = data.get("tracking_truncated", False)
+        # Also detect truncation if user_count > loaded hashes (backward compat)
+        if pattern.user_count > len(pattern._seen_instance_hashes):
+            pattern._tracking_truncated = True
+
+        # Load field semantics (TOIN Evolution)
+        field_semantics_data = data.get("field_semantics", {})
+        if field_semantics_data:
+            pattern.field_semantics = {
+                k: FieldSemantics.from_dict(v) for k, v in field_semantics_data.items()
+            }
+
+        return pattern
+
+
+@dataclass
+class CompressionHint:
+    """Recommendation for how to compress a specific tool output.
+
+    This is what TOIN returns when asked for advice before compression.
+    """
+
+    # Should we compress at all?
+    skip_compression: bool = False
+
+    # How aggressively to compress
+    max_items: int = 20
+    compression_level: Literal["none", "conservative", "moderate", "aggressive"] = "moderate"
+
+    # Which fields to preserve (never remove)
+    preserve_fields: list[str] = field(default_factory=list)
+
+    # Which strategy to use
+    recommended_strategy: str = "default"
+
+    # Why this recommendation
+    reason: str = ""
+    confidence: float = 0.0
+
+    # Source of recommendation
+    source: Literal["network", "local", "default"] = "default"
+    based_on_samples: int = 0
+
+    # === TOIN Evolution: Learned Field Semantics ===
+    # These enable zero-latency signal detection in SmartCrusher.
+    # field_hash -> FieldSemantics (learned semantic type, important values, etc.)
+    field_semantics: dict[str, FieldSemantics] = field(default_factory=dict)
+
+
+@dataclass
+class TOINConfig:
+    """Configuration for the Tool Output Intelligence Network."""
+
+    # Enable/disable TOIN
+    enabled: bool = True
+
+    # Storage
+    storage_path: str | None = None  # Path to store TOIN data
+    auto_save_interval: int = 600  # Auto-save every 10 minutes
+
+    # Network learning thresholds
+    min_samples_for_recommendation: int = 10
+    min_users_for_network_effect: int = 3
+
+    # Recommendation thresholds
+    high_retrieval_threshold: float = 0.5  # Above this = compress less
+    medium_retrieval_threshold: float = 0.2  # Between medium and high = moderate
+
+    # Privacy
+    anonymize_queries: bool = True
+    max_query_patterns: int = 10
+
+    # LOW FIX #22: Metrics/monitoring hooks
+    # Callback for emitting metrics events. Signature: (event_name, event_data) -> None
+    # Event names: "toin.compression", "toin.retrieval", "toin.recommendation", "toin.save"
+    # This allows integration with Prometheus, StatsD, OpenTelemetry, etc.
+    metrics_callback: MetricsCallback | None = None
+
+
+class ToolIntelligenceNetwork:
+    """Aggregates tool patterns across all Headroom users.
+
+    This is the brain of TOIN. It maintains a database of learned patterns
+    for different tool types and provides recommendations based on
+    cross-user intelligence.
+
+    Thread-safe for concurrent access.
+    """
+
+    def __init__(self, config: TOINConfig | None = None):
+        """Initialize TOIN.
+
+        Args:
+            config: Configuration options.
+        """
+        self._config = config or TOINConfig()
+        self._lock = threading.RLock()  # RLock for reentrant locking (save calls export_patterns)
+
+        # Pattern database: structure_hash -> ToolPattern
+        self._patterns: dict[str, ToolPattern] = {}
+
+        # Instance ID for user counting (anonymized)
+        # IMPORTANT: Must be STABLE across restarts to avoid false user count inflation
+        # Derive from storage path if available, otherwise use machine-specific ID
+        self._instance_id = self._generate_stable_instance_id()
+
+        # Tracking
+        self._last_save_time = time.time()
+        self._dirty = False
+
+        # Load existing data
+        if self._config.storage_path:
+            self._load_from_disk()
+
+    def _generate_stable_instance_id(self) -> str:
+        """Generate a stable instance ID that doesn't change across restarts.
+
+        Uses storage path if available, otherwise uses machine-specific info.
+        This prevents false user count inflation when reloading from disk.
+
+        HIGH FIX: Instance ID collision risk
+        Previously used SHA256[:8] (32 bits) which has 50% collision probability
+        at sqrt(2^32) ≈ 65,536 users (birthday paradox). Increased to SHA256[:16]
+        (64 bits) for 50% collision at ~4 billion users, which is acceptable.
+        """
+        if self._config.storage_path:
+            # Derive from storage path - same path = same instance
+            return hashlib.sha256(self._config.storage_path.encode()).hexdigest()[
+                :16
+            ]  # HIGH FIX: 64 bits instead of 32
+        else:
+            # No storage - use a combination of hostname and process info
+            # This is less stable but better than pure random
+            import os
+            import socket
+
+            machine_info = (
+                f"{socket.gethostname()}:{os.getuid() if hasattr(os, 'getuid') else 'unknown'}"
+            )
+            return hashlib.sha256(machine_info.encode()).hexdigest()[:16]  # HIGH FIX: 64 bits
+
+    def _emit_metric(self, event_name: str, event_data: dict[str, Any]) -> None:
+        """Emit a metrics event via the configured callback.
+
+        LOW FIX #22: Provides monitoring integration for external metrics systems.
+
+        Args:
+            event_name: Name of the event (e.g., "toin.compression").
+            event_data: Dictionary of event data to emit.
+        """
+        if self._config.metrics_callback is not None:
+            try:
+                self._config.metrics_callback(event_name, event_data)
+            except Exception as e:
+                # Never let metrics callback failures break TOIN
+                logger.debug(f"Metrics callback failed for {event_name}: {e}")
+
+    def record_compression(
+        self,
+        tool_signature: ToolSignature,
+        original_count: int,
+        compressed_count: int,
+        original_tokens: int,
+        compressed_tokens: int,
+        strategy: str,
+        query_context: str | None = None,
+        items: list[dict[str, Any]] | None = None,
+    ) -> None:
+        """Record a compression event.
+
+        Called after SmartCrusher compresses data. Updates the pattern
+        for this tool type.
+
+        TOIN Evolution: When items are provided, we capture field statistics
+        for learning semantic types (uniqueness, default values, etc.).
+
+        Args:
+            tool_signature: Signature of the tool output structure.
+            original_count: Original number of items.
+            compressed_count: Number of items after compression.
+            original_tokens: Original token count.
+            compressed_tokens: Compressed token count.
+            strategy: Compression strategy used.
+            query_context: Optional user query that triggered this tool call.
+            items: Optional list of items being compressed for field-level learning.
+        """
+        # HIGH FIX: Check enabled FIRST to avoid computing structure_hash if disabled
+        # This saves CPU when TOIN is turned off
+        if not self._config.enabled:
+            return
+
+        # Computing structure_hash can be expensive for large structures
+        sig_hash = tool_signature.structure_hash
+
+        # LOW FIX #22: Emit compression metric
+        self._emit_metric(
+            "toin.compression",
+            {
+                "signature_hash": sig_hash,
+                "original_count": original_count,
+                "compressed_count": compressed_count,
+                "original_tokens": original_tokens,
+                "compressed_tokens": compressed_tokens,
+                "strategy": strategy,
+                "compression_ratio": compressed_count / original_count if original_count > 0 else 0,
+            },
+        )
+
+        with self._lock:
+            # Get or create pattern
+            if sig_hash not in self._patterns:
+                self._patterns[sig_hash] = ToolPattern(tool_signature_hash=sig_hash)
+
+            pattern = self._patterns[sig_hash]
+
+            # Update compression stats
+            pattern.total_compressions += 1
+            pattern.total_items_seen += original_count
+            pattern.total_items_kept += compressed_count
+            pattern.sample_size += 1
+
+            # Update rolling averages
+            n = pattern.total_compressions
+            compression_ratio = compressed_count / original_count if original_count > 0 else 0.0
+            token_reduction = (
+                1 - (compressed_tokens / original_tokens) if original_tokens > 0 else 0.0
+            )
+
+            pattern.avg_compression_ratio = (
+                pattern.avg_compression_ratio * (n - 1) + compression_ratio
+            ) / n
+            pattern.avg_token_reduction = (
+                pattern.avg_token_reduction * (n - 1) + token_reduction
+            ) / n
+
+            # Update strategy stats
+            if strategy not in pattern.strategy_success_rates:
+                pattern.strategy_success_rates[strategy] = 1.0  # Start optimistic
+            else:
+                # Give a small boost for each compression without retrieval
+                # This counteracts the penalty from record_retrieval() and prevents
+                # all strategies from trending to 0.0 over time (one-way ratchet fix)
+                # The boost is small (0.02) because retrieval penalties are larger (0.05-0.15)
+                # This means strategies that cause retrievals will still trend down
+                current_rate = pattern.strategy_success_rates[strategy]
+                pattern.strategy_success_rates[strategy] = min(1.0, current_rate + 0.02)
+
+            # HIGH FIX: Bound strategy_success_rates to prevent unbounded growth
+            # Keep top 20 strategies by success rate
+            if len(pattern.strategy_success_rates) > 20:
+                sorted_strategies = sorted(
+                    pattern.strategy_success_rates.items(),
+                    key=lambda x: x[1],
+                    reverse=True,
+                )[:20]
+                pattern.strategy_success_rates = dict(sorted_strategies)
+
+            # Track unique users via instance_id
+            # FIX: Use _all_seen_instances set for lookup to prevent double-counting
+            # after the storage list hits its cap
+            # CRITICAL FIX #1: Check cap before adding to prevent OOM
+            if self._instance_id not in pattern._all_seen_instances:
+                # CRITICAL FIX: Check if we can verify this is a new user
+                # If tracking was truncated (users lost after restart), we can only
+                # count new users if we can add them to _all_seen_instances for dedup
+                can_track = len(pattern._all_seen_instances) < ToolPattern.MAX_SEEN_INSTANCES
+
+                if can_track:
+                    # Add to the lookup set - we can verify this is new
+                    pattern._all_seen_instances.add(self._instance_id)
+                    # Also add to storage list (capped at 100 for serialization)
+                    if len(pattern._seen_instance_hashes) < 100:
+                        pattern._seen_instance_hashes.append(self._instance_id)
+                    # Safe to increment user_count - we verified it's new
+                    pattern.user_count += 1
+                elif not pattern._tracking_truncated:
+                    # Tracking set is full but we weren't truncated before
+                    # This is a truly new user beyond our tracking capacity
+                    pattern.user_count += 1
+                # else: Can't verify if new, skip incrementing to prevent double-count
+
+            # Track query context patterns for learning (privacy-preserving)
+            if query_context and len(query_context) >= 3:
+                # Normalize and anonymize: extract keywords, remove values
+                query_pattern = self._anonymize_query_pattern(query_context)
+                if query_pattern:
+                    # MEDIUM FIX #10: Track frequency to keep most common patterns
+                    pattern.query_pattern_frequency[query_pattern] = (
+                        pattern.query_pattern_frequency.get(query_pattern, 0) + 1
+                    )
+                    # Update the list to contain top patterns by frequency
+                    if query_pattern not in pattern.common_query_patterns:
+                        pattern.common_query_patterns.append(query_pattern)
+                    # Keep only the most common patterns (by frequency)
+                    if len(pattern.common_query_patterns) > self._config.max_query_patterns:
+                        pattern.common_query_patterns = sorted(
+                            pattern.common_query_patterns,
+                            key=lambda p: pattern.query_pattern_frequency.get(p, 0),
+                            reverse=True,
+                        )[: self._config.max_query_patterns]
+                    # Also limit the frequency dict
+                    if len(pattern.query_pattern_frequency) > self._config.max_query_patterns * 2:
+                        top_patterns = sorted(
+                            pattern.query_pattern_frequency.items(),
+                            key=lambda x: x[1],
+                            reverse=True,
+                        )[: self._config.max_query_patterns * 2]
+                        pattern.query_pattern_frequency = dict(top_patterns)
+
+            # Periodically update recommendations even without retrievals
+            # This ensures optimal_strategy is updated based on success rates
+            if pattern.total_compressions % 10 == 0:
+                self._update_recommendations(pattern)
+
+            # === TOIN Evolution: Field Statistics for Semantic Learning ===
+            # Capture field-level statistics to learn default values and uniqueness
+            if items:
+                self._update_field_statistics(pattern, items)
+
+            pattern.last_updated = time.time()
+            pattern.confidence = self._calculate_confidence(pattern)
+            self._dirty = True
+
+        # Auto-save if needed (outside lock)
+        self._maybe_auto_save()
+
+    def _update_field_statistics(
+        self,
+        pattern: ToolPattern,
+        items: list[dict[str, Any]],
+    ) -> None:
+        """Update field statistics from compression items.
+
+        Captures uniqueness, default values, and value distribution for
+        learning field semantic types.
+
+        Args:
+            pattern: ToolPattern to update.
+            items: Items being compressed.
+        """
+        if not items:
+            return
+
+        # Analyze field statistics (sample up to 100 items to limit CPU)
+        sample_items = items[:100] if len(items) > 100 else items
+
+        # Collect values for each field
+        field_values: dict[str, list[str]] = {}  # field_hash -> list of value_hashes
+
+        for item in sample_items:
+            if not isinstance(item, dict):
+                continue
+
+            for field_name, value in item.items():
+                field_hash = self._hash_field_name(field_name)
+                value_hash = self._hash_value(value)
+
+                if field_hash not in field_values:
+                    field_values[field_hash] = []
+                field_values[field_hash].append(value_hash)
+
+        # Update FieldSemantics with statistics
+        for field_hash, values in field_values.items():
+            if not values:
+                continue
+
+            # Get or create FieldSemantics
+            if field_hash not in pattern.field_semantics:
+                pattern.field_semantics[field_hash] = FieldSemantics(field_hash=field_hash)
+
+            field_sem = pattern.field_semantics[field_hash]
+
+            # Calculate statistics
+            unique_values = len(set(values))
+            total_values = len(values)
+
+            # Find most common value
+            from collections import Counter
+
+            value_counts = Counter(values)
+            most_common_value, most_common_count = value_counts.most_common(1)[0]
+            most_common_frequency = most_common_count / total_values if total_values > 0 else 0.0
+
+            # Record compression stats
+            field_sem.record_compression_stats(
+                unique_values=unique_values,
+                total_values=total_values,
+                most_common_value_hash=most_common_value,
+                most_common_frequency=most_common_frequency,
+            )
+
+        # Bound field_semantics to prevent unbounded growth (max 100 fields)
+        if len(pattern.field_semantics) > 100:
+            # Keep fields with highest activity (retrieval + compression count)
+            sorted_fields = sorted(
+                pattern.field_semantics.items(),
+                key=lambda x: x[1].retrieval_count + x[1].compression_count,
+                reverse=True,
+            )[:100]
+            pattern.field_semantics = dict(sorted_fields)
+
+    def record_retrieval(
+        self,
+        tool_signature_hash: str,
+        retrieval_type: str,
+        query: str | None = None,
+        query_fields: list[str] | None = None,
+        strategy: str | None = None,
+        retrieved_items: list[dict[str, Any]] | None = None,
+    ) -> None:
+        """Record a retrieval event.
+
+        Called when LLM retrieves compressed content. This is the key
+        feedback signal - it means compression was too aggressive.
+
+        TOIN Evolution: When retrieved_items are provided, we learn field
+        semantics from the values. This enables zero-latency signal detection.
+
+        Args:
+            tool_signature_hash: Hash of the tool signature.
+            retrieval_type: "full" or "search".
+            query: Optional search query (will be anonymized).
+            query_fields: Fields mentioned in query (will be hashed).
+            strategy: Compression strategy that was used (for success rate tracking).
+            retrieved_items: Optional list of retrieved items for field-level learning.
+        """
+        if not self._config.enabled:
+            return
+
+        # LOW FIX #22: Emit retrieval metric
+        self._emit_metric(
+            "toin.retrieval",
+            {
+                "signature_hash": tool_signature_hash,
+                "retrieval_type": retrieval_type,
+                "has_query": query is not None,
+                "query_fields_count": len(query_fields) if query_fields else 0,
+                "strategy": strategy,
+            },
+        )
+
+        with self._lock:
+            if tool_signature_hash not in self._patterns:
+                # First time seeing this tool via retrieval
+                self._patterns[tool_signature_hash] = ToolPattern(
+                    tool_signature_hash=tool_signature_hash
+                )
+
+            pattern = self._patterns[tool_signature_hash]
+
+            # Update retrieval stats
+            pattern.total_retrievals += 1
+            if retrieval_type == "full":
+                pattern.full_retrievals += 1
+            else:
+                pattern.search_retrievals += 1
+
+            # Update strategy success rates - retrieval means the strategy was TOO aggressive
+            # Decrease success rate for this strategy
+            if strategy and strategy in pattern.strategy_success_rates:
+                # Exponential moving average: penalize strategies that trigger retrieval
+                # Full retrievals are worse than search retrievals
+                penalty = 0.15 if retrieval_type == "full" else 0.05
+                current_rate = pattern.strategy_success_rates[strategy]
+                pattern.strategy_success_rates[strategy] = max(0.0, current_rate - penalty)
+
+            # Track queried fields (anonymized)
+            if query_fields:
+                for field_name in query_fields:
+                    field_hash = self._hash_field_name(field_name)
+                    pattern.field_retrieval_frequency[field_hash] = (
+                        pattern.field_retrieval_frequency.get(field_hash, 0) + 1
+                    )
+
+                    # Update commonly retrieved fields
+                    if field_hash not in pattern.commonly_retrieved_fields:
+                        # Add if frequently retrieved (check count from dict)
+                        freq = pattern.field_retrieval_frequency.get(field_hash, 0)
+                        if freq >= 3:
+                            pattern.commonly_retrieved_fields.append(field_hash)
+                            # HIGH: Limit commonly_retrieved_fields to prevent unbounded growth
+                            if len(pattern.commonly_retrieved_fields) > 20:
+                                # Keep only the most frequently retrieved fields
+                                sorted_fields = sorted(
+                                    pattern.commonly_retrieved_fields,
+                                    key=lambda f: pattern.field_retrieval_frequency.get(f, 0),
+                                    reverse=True,
+                                )
+                                pattern.commonly_retrieved_fields = sorted_fields[:20]
+
+                # HIGH: Limit field_retrieval_frequency dict to prevent unbounded growth
+                if len(pattern.field_retrieval_frequency) > 100:
+                    sorted_freq_items = sorted(
+                        pattern.field_retrieval_frequency.items(),
+                        key=lambda x: x[1],
+                        reverse=True,
+                    )[:100]
+                    pattern.field_retrieval_frequency = dict(sorted_freq_items)
+
+            # Track query patterns (anonymized)
+            if query and self._config.anonymize_queries:
+                query_pattern = self._anonymize_query_pattern(query)
+                if query_pattern:
+                    # MEDIUM FIX #10: Track frequency to keep most common patterns
+                    pattern.query_pattern_frequency[query_pattern] = (
+                        pattern.query_pattern_frequency.get(query_pattern, 0) + 1
+                    )
+                    if query_pattern not in pattern.common_query_patterns:
+                        pattern.common_query_patterns.append(query_pattern)
+                    # Keep only the most common patterns (by frequency)
+                    if len(pattern.common_query_patterns) > self._config.max_query_patterns:
+                        pattern.common_query_patterns = sorted(
+                            pattern.common_query_patterns,
+                            key=lambda p: pattern.query_pattern_frequency.get(p, 0),
+                            reverse=True,
+                        )[: self._config.max_query_patterns]
+
+            # === TOIN Evolution: Field-Level Semantic Learning ===
+            # Learn from retrieved items to build zero-latency signal detection
+            if retrieved_items:
+                # Extract query operator from query string (for learning)
+                query_operator = self._extract_query_operator(query) if query else "="
+
+                for item in retrieved_items:
+                    if not isinstance(item, dict):
+                        continue
+
+                    for field_name, value in item.items():
+                        field_hash = self._hash_field_name(field_name)
+
+                        # Get or create FieldSemantics for this field
+                        if field_hash not in pattern.field_semantics:
+                            pattern.field_semantics[field_hash] = FieldSemantics(
+                                field_hash=field_hash
+                            )
+
+                        field_sem = pattern.field_semantics[field_hash]
+
+                        # Hash the value for privacy
+                        value_hash = self._hash_value(value)
+
+                        # Record this retrieval
+                        field_sem.record_retrieval_value(value_hash, query_operator)
+
+                # Periodically infer types (every 5 retrievals to save CPU)
+                if pattern.total_retrievals % 5 == 0:
+                    for field_sem in pattern.field_semantics.values():
+                        if field_sem.retrieval_count >= 3:  # Need minimum data
+                            field_sem.infer_type()
+
+                # Bound field_semantics to prevent unbounded growth (max 100 fields)
+                if len(pattern.field_semantics) > 100:
+                    # Keep fields with highest retrieval counts
+                    sorted_semantics = sorted(
+                        pattern.field_semantics.items(),
+                        key=lambda x: x[1].retrieval_count,
+                        reverse=True,
+                    )[:100]
+                    pattern.field_semantics = dict(sorted_semantics)
+
+            # Update recommendations based on new retrieval data
+            self._update_recommendations(pattern)
+
+            pattern.last_updated = time.time()
+            self._dirty = True
+
+        self._maybe_auto_save()
+
+    def get_recommendation(
+        self,
+        tool_signature: ToolSignature,
+        query_context: str | None = None,
+    ) -> CompressionHint:
+        """Get compression recommendation for a tool output.
+
+        This is the main API for SmartCrusher to consult before compressing.
+
+        Args:
+            tool_signature: Signature of the tool output structure.
+            query_context: User query for context-aware recommendations.
+
+        Returns:
+            CompressionHint with recommendations.
+        """
+        if not self._config.enabled:
+            return CompressionHint(source="default", reason="TOIN disabled")
+
+        sig_hash = tool_signature.structure_hash
+
+        with self._lock:
+            pattern = self._patterns.get(sig_hash)
+
+            if pattern is None:
+                # No data for this tool type
+                return CompressionHint(
+                    source="default",
+                    reason="No pattern data for this tool type",
+                )
+
+            # Not enough samples for reliable recommendation
+            if pattern.sample_size < self._config.min_samples_for_recommendation:
+                hint = CompressionHint(
+                    source="local",
+                    reason=f"Only {pattern.sample_size} samples (need {self._config.min_samples_for_recommendation})",
+                    confidence=pattern.confidence,
+                    based_on_samples=pattern.sample_size,
+                )
+                # LOW FIX #22: Emit recommendation metric
+                self._emit_metric(
+                    "toin.recommendation",
+                    {
+                        "signature_hash": sig_hash,
+                        "source": hint.source,
+                        "confidence": hint.confidence,
+                        "skip_compression": hint.skip_compression,
+                        "max_items": hint.max_items,
+                        "compression_level": hint.compression_level,
+                        "based_on_samples": hint.based_on_samples,
+                    },
+                )
+                return hint
+
+            # Build recommendation based on learned patterns
+            hint = self._build_recommendation(pattern, query_context)
+
+            # LOW FIX #22: Emit recommendation metric
+            self._emit_metric(
+                "toin.recommendation",
+                {
+                    "signature_hash": sig_hash,
+                    "source": hint.source,
+                    "confidence": hint.confidence,
+                    "skip_compression": hint.skip_compression,
+                    "max_items": hint.max_items,
+                    "compression_level": hint.compression_level,
+                    "based_on_samples": hint.based_on_samples,
+                },
+            )
+            return hint
+
+    def _build_recommendation(
+        self,
+        pattern: ToolPattern,
+        query_context: str | None,
+    ) -> CompressionHint:
+        """Build a recommendation based on pattern data and query context."""
+        hint = CompressionHint(
+            source="network"
+            if pattern.user_count >= self._config.min_users_for_network_effect
+            else "local",
+            confidence=pattern.confidence,
+            based_on_samples=pattern.sample_size,
+        )
+
+        retrieval_rate = pattern.retrieval_rate
+        full_retrieval_rate = pattern.full_retrieval_rate
+
+        # High retrieval rate = compression too aggressive
+        if retrieval_rate > self._config.high_retrieval_threshold:
+            if full_retrieval_rate > 0.8:
+                # Almost all retrievals are full = don't compress
+                hint.skip_compression = True
+                hint.compression_level = "none"
+                hint.reason = f"Very high full retrieval rate ({full_retrieval_rate:.1%})"
+            else:
+                # High retrieval but mostly search = compress conservatively
+                hint.max_items = pattern.optimal_max_items
+                hint.compression_level = "conservative"
+                hint.reason = f"High retrieval rate ({retrieval_rate:.1%})"
+
+        elif retrieval_rate > self._config.medium_retrieval_threshold:
+            # Moderate retrieval = moderate compression
+            hint.max_items = max(20, pattern.optimal_max_items)
+            hint.compression_level = "moderate"
+            hint.reason = f"Moderate retrieval rate ({retrieval_rate:.1%})"
+
+        else:
+            # Low retrieval = aggressive compression works
+            hint.max_items = min(15, pattern.optimal_max_items)
+            hint.compression_level = "aggressive"
+            hint.reason = f"Low retrieval rate ({retrieval_rate:.1%})"
+
+        # Build preserve_fields list weighted by retrieval frequency
+        # Start with pattern's preserve_fields, then enhance based on query
+        preserve_fields = pattern.preserve_fields.copy()
+        query_fields_count = 0
+
+        # If we have query context, extract field names and prioritize them
+        if query_context and pattern.field_retrieval_frequency:
+            # Extract field names from query context
+            import re
+
+            query_field_names = re.findall(r"(\w+)[=:]", query_context.lower())
+
+            # Hash them and check if they're in our frequency data
+            for field_name in query_field_names:
+                field_hash = self._hash_field_name(field_name)
+                if field_hash in pattern.field_retrieval_frequency:
+                    # This field is known to be retrieved - prioritize it
+                    if field_hash in preserve_fields:
+                        # Move to front
+                        preserve_fields.remove(field_hash)
+                    preserve_fields.insert(0, field_hash)
+                    query_fields_count += 1
+
+        # Sort remaining fields by retrieval frequency (most frequent first)
+        if pattern.field_retrieval_frequency and len(preserve_fields) > 1:
+            # Separate query-mentioned fields (already at front) from others
+            if query_fields_count < len(preserve_fields):
+                rest = preserve_fields[query_fields_count:]
+                rest.sort(
+                    key=lambda f: pattern.field_retrieval_frequency.get(f, 0),
+                    reverse=True,
+                )
+                preserve_fields = preserve_fields[:query_fields_count] + rest
+
+        hint.preserve_fields = preserve_fields[:10]  # Limit to top 10
+
+        # Use optimal strategy if known AND it has good success rate
+        if pattern.optimal_strategy != "default":
+            success_rate = pattern.strategy_success_rates.get(pattern.optimal_strategy, 1.0)
+            # Only recommend strategy if success rate >= 0.5
+            # Lower success rates mean this strategy often causes retrievals
+            if success_rate >= 0.5:
+                hint.recommended_strategy = pattern.optimal_strategy
+            else:
+                # Strategy has poor success rate - reduce confidence
+                hint.confidence *= success_rate
+                hint.reason += (
+                    f" (strategy {pattern.optimal_strategy} has low success: {success_rate:.1%})"
+                )
+                # Try to find a better strategy
+                best_strategy = self._find_best_strategy(pattern)
+                if best_strategy and best_strategy != pattern.optimal_strategy:
+                    hint.recommended_strategy = best_strategy
+                    hint.reason += f", using {best_strategy} instead"
+
+        # Boost max_items if query_context matches common retrieval patterns
+        # This prevents unnecessary retrieval when we can predict what's needed
+        if query_context:
+            query_lower = query_context.lower()
+
+            # Check for exhaustive query keywords that suggest user needs all data
+            exhaustive_keywords = ["all", "every", "complete", "full", "entire", "list all"]
+            if any(kw in query_lower for kw in exhaustive_keywords):
+                # User likely needs more data - be conservative
+                hint.max_items = max(hint.max_items, 40)
+                hint.compression_level = "conservative"
+                hint.reason += " (exhaustive query detected)"
+
+            # Check against common retrieval patterns
+            if pattern.common_query_patterns:
+                query_pattern = self._anonymize_query_pattern(query_context)
+                if query_pattern:
+                    # Exact match
+                    if query_pattern in pattern.common_query_patterns:
+                        hint.max_items = max(hint.max_items, 30)
+                        hint.reason += " (query matches retrieval pattern)"
+                    else:
+                        # Partial match: check if any stored pattern is contained in query
+                        for stored_pattern in pattern.common_query_patterns:
+                            # Check if key fields match (e.g., "status:*" in both)
+                            stored_fields = {
+                                f.split(":")[0] for f in stored_pattern.split() if ":" in f
+                            }
+                            query_fields = {
+                                f.split(":")[0] for f in query_pattern.split() if ":" in f
+                            }
+                            # If query uses same fields as a problematic pattern, be conservative
+                            if stored_fields and stored_fields.issubset(query_fields):
+                                hint.max_items = max(hint.max_items, 25)
+                                hint.reason += " (query uses fields from retrieval pattern)"
+                                break
+
+        # === TOIN Evolution: Include learned field semantics ===
+        # Copy field_semantics with sufficient confidence for SmartCrusher to use
+        # Only include fields with confidence >= 0.3 to reduce noise
+        if pattern.field_semantics:
+            hint.field_semantics = {
+                field_hash: field_sem
+                for field_hash, field_sem in pattern.field_semantics.items()
+                if field_sem.confidence >= 0.3 or field_sem.retrieval_count >= 3
+            }
+
+        return hint
+
+    def _find_best_strategy(self, pattern: ToolPattern) -> str | None:
+        """Find the strategy with the best success rate.
+
+        Returns None if no strategies have been tried or all have low success.
+        """
+        if not pattern.strategy_success_rates:
+            return None
+
+        # Find strategy with highest success rate above threshold
+        best_strategy = None
+        best_rate = 0.5  # Minimum acceptable rate
+
+        for strategy, rate in pattern.strategy_success_rates.items():
+            if rate > best_rate:
+                best_rate = rate
+                best_strategy = strategy
+
+        return best_strategy
+
+    def _update_recommendations(self, pattern: ToolPattern) -> None:
+        """Update learned recommendations for a pattern."""
+        # Calculate optimal max_items based on retrieval rate
+        retrieval_rate = pattern.retrieval_rate
+
+        if retrieval_rate > self._config.high_retrieval_threshold:
+            if pattern.full_retrieval_rate > 0.8:
+                pattern.skip_compression_recommended = True
+                pattern.optimal_max_items = pattern.total_items_seen // max(
+                    1, pattern.total_compressions
+                )
+            else:
+                pattern.optimal_max_items = 50
+        elif retrieval_rate > self._config.medium_retrieval_threshold:
+            pattern.optimal_max_items = 30
+        else:
+            pattern.optimal_max_items = 20
+
+        # Update preserve_fields from frequently retrieved fields
+        if pattern.field_retrieval_frequency:
+            # Get top 5 most retrieved fields
+            sorted_fields = sorted(
+                pattern.field_retrieval_frequency.items(),
+                key=lambda x: x[1],
+                reverse=True,
+            )[:5]
+            pattern.preserve_fields = [f for f, _ in sorted_fields]
+
+        # Update optimal strategy (pick most successful)
+        if pattern.strategy_success_rates:
+            best_strategy = max(
+                pattern.strategy_success_rates.items(),
+                key=lambda x: x[1],
+            )[0]
+            pattern.optimal_strategy = best_strategy
+
+    def _calculate_confidence(self, pattern: ToolPattern) -> float:
+        """Calculate confidence level for a pattern."""
+        # Base confidence on sample size
+        sample_confidence = min(0.7, pattern.sample_size / 100)
+
+        # Boost if from multiple users
+        # FIX: Changed from `user_count / 10 * 0.1` (= user_count * 0.01, too small)
+        # to `user_count * 0.03` for meaningful boost at low user counts
+        # - 3 users: 0.09 boost
+        # - 10 users: 0.30 boost (capped)
+        user_boost = 0.0
+        if pattern.user_count >= self._config.min_users_for_network_effect:
+            user_boost = min(0.3, pattern.user_count * 0.03)
+
+        return min(0.95, sample_confidence + user_boost)
+
+    def _hash_field_name(self, field_name: str) -> str:
+        """Hash a field name for anonymization."""
+        return hashlib.sha256(field_name.encode()).hexdigest()[:8]
+
+    def _anonymize_query_pattern(self, query: str) -> str | None:
+        """Extract anonymized pattern from a query.
+
+        Keeps structural patterns, removes specific values.
+        E.g., "status:error AND user:john" -> "status:* AND user:*"
+        """
+        if not query:
+            return None
+
+        # Simple pattern extraction: replace values after : or =
+        import re
+
+        # Match field:value or field="value" patterns, but don't include spaces in unquoted values
+        pattern = re.sub(r'(\w+)[=:](?:"[^"]*"|\'[^\']*\'|\w+)', r"\1:*", query)
+
+        # Remove if it's just generic
+        if pattern in ("*", ""):
+            return None
+
+        return pattern
+
+    def _hash_value(self, value: Any) -> str:
+        """Hash a value for privacy-preserving storage.
+
+        Handles all types by converting to a canonical string representation.
+        """
+        if value is None:
+            canonical = "null"
+        elif isinstance(value, bool):
+            canonical = "true" if value else "false"
+        elif isinstance(value, (int, float)):
+            canonical = str(value)
+        elif isinstance(value, str):
+            canonical = value
+        elif isinstance(value, (list, dict)):
+            # For complex types, use JSON serialization
+            try:
+                canonical = json.dumps(value, sort_keys=True, default=str)
+            except (TypeError, ValueError):
+                canonical = str(value)
+        else:
+            canonical = str(value)
+
+        return hashlib.sha256(canonical.encode()).hexdigest()[:8]
+
+    def _extract_query_operator(self, query: str) -> str:
+        """Extract the dominant query operator from a search query.
+
+        Used for learning field semantic types from query patterns.
+
+        Returns:
+            Query operator: "=", "!=", ">", "<", ">=", "<=", "contains", or "="
+        """
+        if not query:
+            return "="
+
+        query_lower = query.lower()
+
+        # Check for inequality operators
+        if "!=" in query or " not " in query_lower or " ne " in query_lower:
+            return "!="
+        if ">=" in query or " gte " in query_lower:
+            return ">="
+        if "<=" in query or " lte " in query_lower:
+            return "<="
+        if ">" in query or " gt " in query_lower:
+            return ">"
+        if "<" in query or " lt " in query_lower:
+            return "<"
+
+        # Check for text search operators
+        if " like " in query_lower or " contains " in query_lower or "*" in query:
+            return "contains"
+
+        # Default to equality
+        return "="
+
+    def get_stats(self) -> dict[str, Any]:
+        """Get overall TOIN statistics."""
+        with self._lock:
+            total_compressions = sum(p.total_compressions for p in self._patterns.values())
+            total_retrievals = sum(p.total_retrievals for p in self._patterns.values())
+
+            return {
+                "enabled": self._config.enabled,
+                "patterns_tracked": len(self._patterns),
+                "total_compressions": total_compressions,
+                "total_retrievals": total_retrievals,
+                "global_retrieval_rate": (
+                    total_retrievals / total_compressions if total_compressions > 0 else 0.0
+                ),
+                "patterns_with_recommendations": sum(
+                    1
+                    for p in self._patterns.values()
+                    if p.sample_size >= self._config.min_samples_for_recommendation
+                ),
+            }
+
+    def get_pattern(self, signature_hash: str) -> ToolPattern | None:
+        """Get pattern data for a specific tool signature.
+
+        HIGH FIX: Returns a deep copy to prevent external mutation of internal state.
+        """
+        import copy
+
+        with self._lock:
+            pattern = self._patterns.get(signature_hash)
+            if pattern is not None:
+                return copy.deepcopy(pattern)
+            return None
+
+    def export_patterns(self) -> dict[str, Any]:
+        """Export all patterns for sharing/aggregation."""
+        with self._lock:
+            return {
+                "version": "1.0",
+                "export_timestamp": time.time(),
+                "instance_id": self._instance_id,
+                "patterns": {
+                    sig_hash: pattern.to_dict() for sig_hash, pattern in self._patterns.items()
+                },
+            }
+
+    def import_patterns(self, data: dict[str, Any]) -> None:
+        """Import patterns from another source.
+
+        Used for federated learning: aggregate patterns from multiple
+        Headroom instances without sharing actual data.
+
+        Args:
+            data: Exported pattern data.
+        """
+        if not self._config.enabled:
+            return
+
+        patterns_data = data.get("patterns", {})
+        source_instance = data.get("instance_id", "unknown")
+
+        with self._lock:
+            for sig_hash, pattern_dict in patterns_data.items():
+                imported = ToolPattern.from_dict(pattern_dict)
+
+                if sig_hash in self._patterns:
+                    # Merge with existing
+                    self._merge_patterns(self._patterns[sig_hash], imported)
+                else:
+                    # Add new pattern - need to track source instance
+                    self._patterns[sig_hash] = imported
+
+                    # For NEW patterns from another instance, track the source in
+                    # _seen_instance_hashes so user_count reflects cross-user data
+                    if source_instance != self._instance_id:
+                        pattern = self._patterns[sig_hash]
+                        if source_instance not in pattern._seen_instance_hashes:
+                            # Limit storage to 100 unique instances to bound memory
+                            if len(pattern._seen_instance_hashes) < 100:
+                                pattern._seen_instance_hashes.append(source_instance)
+                            # CRITICAL: Always increment user_count (even after cap)
+                            pattern.user_count += 1
+
+            self._dirty = True
+
+    def _merge_patterns(self, existing: ToolPattern, imported: ToolPattern) -> None:
+        """Merge imported pattern into existing."""
+        total = existing.sample_size + imported.sample_size
+        if total == 0:
+            return
+
+        w_existing = existing.sample_size / total
+        w_imported = imported.sample_size / total
+
+        # Merge counts
+        existing.total_compressions += imported.total_compressions
+        existing.total_retrievals += imported.total_retrievals
+        existing.full_retrievals += imported.full_retrievals
+        existing.search_retrievals += imported.search_retrievals
+        existing.total_items_seen += imported.total_items_seen
+        existing.total_items_kept += imported.total_items_kept
+
+        # Weighted averages
+        existing.avg_compression_ratio = (
+            existing.avg_compression_ratio * w_existing
+            + imported.avg_compression_ratio * w_imported
+        )
+        existing.avg_token_reduction = (
+            existing.avg_token_reduction * w_existing + imported.avg_token_reduction * w_imported
+        )
+
+        # Merge field frequencies
+        for field_hash, count in imported.field_retrieval_frequency.items():
+            existing.field_retrieval_frequency[field_hash] = (
+                existing.field_retrieval_frequency.get(field_hash, 0) + count
+            )
+        # HIGH: Limit field_retrieval_frequency dict to prevent unbounded growth
+        if len(existing.field_retrieval_frequency) > 100:
+            # Keep only the most frequently retrieved fields
+            sorted_fields = sorted(
+                existing.field_retrieval_frequency.items(),
+                key=lambda x: x[1],
+                reverse=True,
+            )[:100]
+            existing.field_retrieval_frequency = dict(sorted_fields)
+
+        # Merge commonly retrieved fields
+        for field_hash in imported.commonly_retrieved_fields:
+            if field_hash not in existing.commonly_retrieved_fields:
+                existing.commonly_retrieved_fields.append(field_hash)
+        # HIGH: Limit commonly_retrieved_fields to prevent unbounded growth
+        if len(existing.commonly_retrieved_fields) > 20:
+            # Prioritize by retrieval frequency if available
+            if existing.field_retrieval_frequency:
+                existing.commonly_retrieved_fields = sorted(
+                    existing.commonly_retrieved_fields,
+                    key=lambda f: existing.field_retrieval_frequency.get(f, 0),
+                    reverse=True,
+                )[:20]
+            else:
+                existing.commonly_retrieved_fields = existing.commonly_retrieved_fields[:20]
+
+        # Merge query patterns (for federated learning)
+        # MEDIUM FIX #10: Also merge query_pattern_frequency for proper ranking
+        for query_pattern, freq in imported.query_pattern_frequency.items():
+            existing.query_pattern_frequency[query_pattern] = (
+                existing.query_pattern_frequency.get(query_pattern, 0) + freq
+            )
+        for query_pattern in imported.common_query_patterns:
+            if query_pattern not in existing.common_query_patterns:
+                existing.common_query_patterns.append(query_pattern)
+        # Keep only the most common patterns (by frequency)
+        if len(existing.common_query_patterns) > self._config.max_query_patterns:
+            existing.common_query_patterns = sorted(
+                existing.common_query_patterns,
+                key=lambda p: existing.query_pattern_frequency.get(p, 0),
+                reverse=True,
+            )[: self._config.max_query_patterns]
+        # Limit frequency dict
+        if len(existing.query_pattern_frequency) > self._config.max_query_patterns * 2:
+            top_patterns = sorted(
+                existing.query_pattern_frequency.items(),
+                key=lambda x: x[1],
+                reverse=True,
+            )[: self._config.max_query_patterns * 2]
+            existing.query_pattern_frequency = dict(top_patterns)
+
+        # Merge strategy success rates (weighted average)
+        for strategy, rate in imported.strategy_success_rates.items():
+            if strategy in existing.strategy_success_rates:
+                existing.strategy_success_rates[strategy] = (
+                    existing.strategy_success_rates[strategy] * w_existing + rate * w_imported
+                )
+            else:
+                existing.strategy_success_rates[strategy] = rate
+
+        # HIGH FIX: Bound strategy_success_rates after merge
+        if len(existing.strategy_success_rates) > 20:
+            sorted_strategies = sorted(
+                existing.strategy_success_rates.items(),
+                key=lambda x: x[1],
+                reverse=True,
+            )[:20]
+            existing.strategy_success_rates = dict(sorted_strategies)
+
+        # Merge preserve_fields (union of both, deduplicated)
+        for preserve_field in imported.preserve_fields:
+            if preserve_field not in existing.preserve_fields:
+                existing.preserve_fields.append(preserve_field)
+        # Keep only top 10 most important fields
+        if len(existing.preserve_fields) > 10:
+            # Prioritize by retrieval frequency if available
+            if existing.field_retrieval_frequency:
+                existing.preserve_fields = sorted(
+                    existing.preserve_fields,
+                    key=lambda f: existing.field_retrieval_frequency.get(f, 0),
+                    reverse=True,
+                )[:10]
+            else:
+                existing.preserve_fields = existing.preserve_fields[:10]
+
+        # Merge skip_compression_recommended (true if either recommends skip)
+        if imported.skip_compression_recommended:
+            # Imported has more data suggesting skip - consider it
+            if imported.sample_size > existing.sample_size // 2:
+                existing.skip_compression_recommended = True
+
+        # Merge optimal_strategy (prefer the one with better success rate)
+        if imported.optimal_strategy != "default":
+            imported_rate = imported.strategy_success_rates.get(imported.optimal_strategy, 0.5)
+            existing_rate = (
+                existing.strategy_success_rates.get(existing.optimal_strategy, 0.5)
+                if existing.optimal_strategy != "default"
+                else 0.0
+            )
+
+            if imported_rate > existing_rate:
+                existing.optimal_strategy = imported.optimal_strategy
+
+        # Merge optimal_max_items (weighted average with bounds)
+        if imported.optimal_max_items > 0:
+            merged_max_items = int(
+                existing.optimal_max_items * w_existing + imported.optimal_max_items * w_imported
+            )
+            # Ensure valid bounds: min 3 items, max 1000 items
+            existing.optimal_max_items = max(3, min(1000, merged_max_items))
+
+        existing.sample_size = total
+
+        # Merge seen instance hashes (union of both, limited to 100 for storage)
+        # CRITICAL FIX #1 & #3: Simplified user count merge logic with cap enforcement.
+        # user_count is the authoritative count even when sets hit their caps.
+        new_users_found = 0
+        for instance_hash in imported._seen_instance_hashes:
+            # Use _all_seen_instances for deduplication (the authoritative set)
+            if instance_hash not in existing._all_seen_instances:
+                # Add to lookup set (with cap to prevent OOM)
+                if len(existing._all_seen_instances) < ToolPattern.MAX_SEEN_INSTANCES:
+                    existing._all_seen_instances.add(instance_hash)
+                # Limit storage list to 100 unique instances to bound serialization
+                if len(existing._seen_instance_hashes) < 100:
+                    existing._seen_instance_hashes.append(instance_hash)
+                new_users_found += 1
+
+        # Also merge instances from imported._all_seen_instances that weren't in list
+        # (in case imported had more than 100 instances)
+        for instance_hash in imported._all_seen_instances:
+            if instance_hash not in existing._all_seen_instances:
+                # Add with cap check
+                if len(existing._all_seen_instances) < ToolPattern.MAX_SEEN_INSTANCES:
+                    existing._all_seen_instances.add(instance_hash)
+                # Storage list already at limit, just track for dedup
+                new_users_found += 1
+
+        # CRITICAL FIX #3: Simplified user count calculation.
+        # We count new users from both the list and set, then add any users
+        # that imported had beyond what we could deduplicate (when both hit caps).
+        # imported.user_count may be > len(imported._all_seen_instances) if they hit cap
+        users_beyond_imported_tracking = max(
+            0, imported.user_count - len(imported._all_seen_instances)
+        )
+        existing.user_count += new_users_found + users_beyond_imported_tracking
+
+        existing.last_updated = time.time()
+
+        # Recalculate recommendations based on merged data
+        self._update_recommendations(existing)
+
+    def save(self) -> None:
+        """Save TOIN data to disk with atomic write.
+
+        Uses a temporary file and rename to ensure atomicity.
+        If the write fails, the original file is preserved.
+
+        HIGH FIX: Serialize under lock but write outside lock to prevent
+        blocking other threads during slow file I/O.
+        """
+        if not self._config.storage_path:
+            return
+
+        import tempfile
+
+        # Step 1: Serialize under lock (fast in-memory operation)
+        with self._lock:
+            data = self.export_patterns()
+
+        # Step 2: Write outside lock (slow I/O operation)
+        path = Path(self._config.storage_path)
+
+        try:
+            # Create parent directories if needed
+            path.parent.mkdir(parents=True, exist_ok=True)
+
+            # Serialize to string (outside lock but before file ops)
+            json_data = json.dumps(data, indent=2)
+
+            # Write to temporary file first (atomic write pattern)
+            # Use same directory to ensure same filesystem for rename
+            fd, tmp_path = tempfile.mkstemp(dir=path.parent, prefix=".toin_", suffix=".tmp")
+            try:
+                with open(fd, "w") as f:
+                    f.write(json_data)
+
+                # Atomic rename (on POSIX systems)
+                Path(tmp_path).replace(path)
+
+            except Exception:
+                # Clean up temp file on failure
+                try:
+                    Path(tmp_path).unlink()
+                except OSError:
+                    pass
+                raise
+
+            # Step 3: Update state under lock (fast)
+            with self._lock:
+                self._dirty = False
+                self._last_save_time = time.time()
+
+        except OSError as e:
+            # Log error but don't crash - TOIN should be resilient
+            logger.warning(f"Failed to save TOIN data: {e}")
+
+    def _load_from_disk(self) -> None:
+        """Load TOIN data from disk."""
+        if not self._config.storage_path:
+            return
+
+        path = Path(self._config.storage_path)
+        if not path.exists():
+            return
+
+        try:
+            with open(path) as f:
+                data = json.load(f)
+            self.import_patterns(data)
+            self._dirty = False
+        except (json.JSONDecodeError, OSError):
+            pass  # Start fresh if corrupted
+
+    def _maybe_auto_save(self) -> None:
+        """Auto-save if enough time has passed.
+
+        HIGH FIX: Check conditions under lock to prevent race where another
+        thread modifies _dirty or _last_save_time between check and save.
+        The save() method already acquires the lock, and we use RLock so
+        it's safe to hold the lock when calling save().
+        """
+        if not self._config.storage_path or not self._config.auto_save_interval:
+            return
+
+        # Check under lock to prevent race conditions
+        with self._lock:
+            if not self._dirty:
+                return
+
+            elapsed = time.time() - self._last_save_time
+            if elapsed >= self._config.auto_save_interval:
+                # save() uses the same RLock, so this is safe
+                self.save()
+
+    def clear(self) -> None:
+        """Clear all TOIN data. Mainly for testing."""
+        with self._lock:
+            self._patterns.clear()
+            self._dirty = False
+
+
+# Global TOIN instance (lazy initialization)
+_toin_instance: ToolIntelligenceNetwork | None = None
+_toin_lock = threading.Lock()
+
+
+def get_toin(config: TOINConfig | None = None) -> ToolIntelligenceNetwork:
+    """Get the global TOIN instance.
+
+    Thread-safe singleton pattern. Always acquires lock to avoid subtle
+    race conditions in double-checked locking on non-CPython implementations.
+
+    Args:
+        config: Configuration (only used on first call). If the instance
+            already exists, config is ignored and a warning is logged.
+
+    Returns:
+        Global ToolIntelligenceNetwork instance.
+    """
+    global _toin_instance
+
+    # CRITICAL FIX: Always acquire lock for thread safety across all Python
+    # implementations. The overhead is negligible since we only construct once.
+    with _toin_lock:
+        if _toin_instance is None:
+            _toin_instance = ToolIntelligenceNetwork(config)
+        elif config is not None:
+            # Warn when config is silently ignored
+            logger.warning(
+                "TOIN config ignored: instance already exists. "
+                "Call reset_toin() first if you need to change config."
+            )
+
+    return _toin_instance
+
+
+def reset_toin() -> None:
+    """Reset the global TOIN instance. Mainly for testing."""
+    global _toin_instance
+
+    with _toin_lock:
+        if _toin_instance is not None:
+            _toin_instance.clear()
+        _toin_instance = None