omnicache-proxy 2.0.0__py3-none-any.whl

core/config.py

@@ -0,0 +1,62 @@
+"""
+Configuration and pricing registry for OmniCache AI Proxy.
+"""
+
+import os
+from typing import Dict, Any
+
+# Provider Pricing Table (USD per 1,000,000 tokens)
+# Updated to reflect 2025/2026 current provider pricing
+MODEL_PRICING: Dict[str, Dict[str, float]] = {
+    # OpenAI Models
+    "gpt-4o": {"input": 2.50, "output": 10.00, "cached_input": 1.25},
+    "gpt-4o-mini": {"input": 0.15, "output": 0.60, "cached_input": 0.075},
+    "o1": {"input": 15.00, "output": 60.00, "cached_input": 7.50},
+    "o3-mini": {"input": 1.10, "output": 4.40, "cached_input": 0.55},
+    "gpt-4-turbo": {"input": 10.00, "output": 30.00, "cached_input": 5.00},
+    "gpt-3.5-turbo": {"input": 0.50, "output": 1.50, "cached_input": 0.25},
+    
+    # Anthropic Models
+    "claude-3-5-sonnet-20241022": {"input": 3.00, "output": 15.00, "cached_input": 0.30},
+    "claude-3-5-haiku-20241022": {"input": 0.80, "output": 4.00, "cached_input": 0.08},
+    "claude-3-7-sonnet": {"input": 3.00, "output": 15.00, "cached_input": 0.30},
+    
+    # Google Gemini Models
+    "gemini-2.5-flash": {"input": 0.10, "output": 0.40, "cached_input": 0.025},
+    "gemini-1.5-pro": {"input": 1.25, "output": 5.00, "cached_input": 0.3125},
+    "gemini-1.5-flash": {"input": 0.075, "output": 0.30, "cached_input": 0.01875},
+    
+    # Default fallback
+    "default": {"input": 2.00, "output": 8.00, "cached_input": 1.00},
+}
+
+class ProxyConfig:
+    PORT: int = int(os.getenv("OMNICACHE_PORT", "8000"))
+    HOST: str = os.getenv("OMNICACHE_HOST", "0.0.0.0")
+    
+    # Default Upstream Provider endpoints
+    OPENAI_BASE_URL: str = os.getenv("OPENAI_BASE_URL", "https://api.openai.com/v1")
+    ANTHROPIC_BASE_URL: str = os.getenv("ANTHROPIC_BASE_URL", "https://api.anthropic.com/v1")
+    GEMINI_BASE_URL: str = os.getenv("GEMINI_BASE_URL", "https://generativelanguage.googleapis.com/v1beta/openai")
+    
+    # Cache Configuration
+    DEFAULT_SIMILARITY_THRESHOLD: float = float(os.getenv("SIMILARITY_THRESHOLD", "0.92"))
+    EXACT_CACHE_TTL_SECONDS: int = int(os.getenv("EXACT_CACHE_TTL", "604800"))  # 7 days
+    SEMANTIC_CACHE_TTL_SECONDS: int = int(os.getenv("SEMANTIC_CACHE_TTL", "604800"))  # 7 days
+    MAX_CACHE_ENTRIES_PER_TENANT: int = int(os.getenv("MAX_CACHE_ENTRIES", "10000"))
+    
+    # Temperature threshold above which semantic cache is bypassed
+    TEMPERATURE_BYPASS_THRESHOLD: float = 0.7
+    
+    # Token Jitter Stream Velocity (tokens per second for cached stream playback)
+    STREAM_REPLAY_TOKENS_PER_SEC: float = 65.0
+    
+    # SingleFlight lock timeout in seconds
+    SINGLEFLIGHT_TIMEOUT_SECONDS: float = 30.0
+    
+    # Upstream Connection Pool settings
+    HTTP_POOL_MAX_CONNECTIONS: int = 100
+    HTTP_POOL_MAX_KEEPALIVE: int = 20
+    HTTP_TIMEOUT_SECONDS: float = 60.0
+
+config = ProxyConfig()

core/embeddings.py

@@ -0,0 +1,140 @@
+"""
+High-performance in-memory semantic embedding engine.
+Generates normalized dense vector representations for sub-millisecond similarity search.
+"""
+
+import math
+import re
+import hashlib
+from typing import List, Dict, Tuple, Optional
+
+class FastSemanticEmbedder:
+    """
+    Sub-millisecond semantic text embedder using high-dimensional hashed character/word n-gram 
+    content-term frequency projection, synonym canonicalization, and L2-unit normalization.
+    Provides robust semantic matching for question rephrasings, synonyms, and variations.
+    """
+    DIMENSIONS: int = 512
+    
+    # Common English & Multilingual stopwords for query normalization
+    STOPWORDS = {
+        "a", "about", "above", "after", "again", "against", "all", "am", "an", "and", "any", "are", 
+        "as", "at", "be", "because", "been", "before", "being", "below", "between", "both", "but", 
+        "by", "could", "did", "do", "does", "doing", "down", "during", "each", "few", "for", "from", 
+        "further", "had", "has", "have", "having", "he", "her", "here", "hers", "herself", "him", 
+        "himself", "his", "how", "i", "if", "in", "into", "is", "it", "its", "itself", "just", 
+        "me", "more", "most", "my", "myself", "no", "nor", "not", "now", "of", "off", "on", "once", 
+        "only", "or", "other", "ought", "our", "ours", "ourselves", "out", "over", "own", "same", 
+        "she", "should", "so", "some", "such", "than", "that", "the", "their", "theirs", "them", 
+        "themselves", "then", "there", "these", "they", "this", "those", "through", "to", "too", 
+        "under", "until", "up", "very", "was", "we", "were", "what", "when", "where", "which", 
+        "while", "who", "whom", "why", "with", "would", "you", "your", "yours", "yourself", "yourselves",
+        "please", "tell", "explain", "help", "can", "could"
+    }
+
+    SYNONYM_MAP = {
+        "recover": "reset",
+        "recovery": "reset",
+        "forgotten": "reset",
+        "forgot": "reset",
+        "procedure": "steps",
+        "method": "steps",
+        "instructions": "steps",
+        "location": "located",
+        "whereabouts": "located",
+        "place": "located",
+        "pricing": "price",
+        "costs": "price",
+        "rate": "price",
+        "authenticate": "login",
+        "signin": "login",
+        "signup": "register",
+        "terminate": "cancel",
+        "modify": "change",
+        "update": "change",
+        "create": "make",
+        "build": "make",
+        "construct": "make",
+        "generate": "make",
+        "fix": "repair",
+        "troubleshoot": "repair"
+    }
+
+    @classmethod
+    def clean_and_tokenize(cls, text: str) -> List[str]:
+        """Normalize text: lowercase, remove special characters, tokenize, canonicalize synonyms."""
+        text = text.lower()
+        # Keep alphanumeric, remove punctuation
+        text = re.sub(r"[^\w\s]", " ", text)
+        raw_tokens = text.split()
+        return [cls.SYNONYM_MAP.get(t, t) for t in raw_tokens]
+
+    @classmethod
+    def get_features(cls, text: str) -> Dict[str, float]:
+        """Extract unigram, content bigrams, and char n-gram weighted features."""
+        tokens = cls.clean_and_tokenize(text)
+        if not tokens:
+            return {}
+            
+        features: Dict[str, float] = {}
+        content_tokens = [t for t in tokens if t not in cls.STOPWORDS]
+        
+        # 1. Word unigrams (content words get high weight, stopwords low weight)
+        for token in tokens:
+            weight = 0.1 if token in cls.STOPWORDS else 2.0
+            features[f"w:{token}"] = features.get(f"w:{token}", 0.0) + weight
+            
+        # 2. Content bigrams (skip noise stopwords)
+        for i in range(len(content_tokens) - 1):
+            bg = f"{content_tokens[i]}_{content_tokens[i+1]}"
+            features[f"bg:{bg}"] = features.get(f"bg:{bg}", 0.0) + 1.0
+            
+        # 3. Subword 3-grams and 4-grams for content words (typos and morphology)
+        for token in content_tokens:
+            if len(token) >= 3:
+                for n in (3, 4):
+                    for i in range(len(token) - n + 1):
+                        ngram = token[i:i+n]
+                        features[f"ng:{ngram}"] = features.get(f"ng:{ngram}", 0.0) + 0.5
+                        
+        return features
+
+    @classmethod
+    def embed(cls, text: str) -> List[float]:
+        """
+        Embeds text into a 512-dimensional L2-normalized dense vector.
+        """
+        if not text or not text.strip():
+            return [0.0] * cls.DIMENSIONS
+            
+        features = cls.get_features(text)
+        vector = [0.0] * cls.DIMENSIONS
+        
+        # Feature hashing into fixed dimension space
+        for feat, weight in features.items():
+            h = int(hashlib.md5(feat.encode('utf-8')).hexdigest()[:8], 16)
+            idx = h % cls.DIMENSIONS
+            sign = 1.0 if (h >> 4) & 1 else -1.0
+            vector[idx] += sign * weight
+            
+        # Compute L2 Norm (Euclidean length)
+        norm_sq = sum(x * x for x in vector)
+        if norm_sq > 0:
+            norm = math.sqrt(norm_sq)
+            vector = [x / norm for x in vector]
+            
+        return vector
+
+    @staticmethod
+    def cosine_similarity(vec_a: List[float], vec_b: List[float]) -> float:
+        """
+        Calculates cosine similarity between two unit-normalized vectors.
+        For unit vectors: dot_product = cosine_similarity.
+        """
+        if not vec_a or not vec_b or len(vec_a) != len(vec_b):
+            return 0.0
+            
+        # Dot product
+        dot = sum(a * b for a, b in zip(vec_a, vec_b))
+        # Clamp to [0.0, 1.0] for similarity index
+        return max(0.0, min(1.0, dot))

core/hasher.py

@@ -0,0 +1,105 @@
+"""
+Composite hashing and prompt extraction utilities.
+Ensures zero collisions between differing JSON schemas, system prompts, or tool definitions.
+Also includes optional PII redaction utilities for enterprise privacy compliance.
+"""
+
+import hashlib
+import json
+import re
+from typing import Dict, Any, Tuple, Optional, List
+
+class RequestHasher:
+    # Common PII Regex Patterns
+    SSN_PATTERN = r"\b\d{3}-\d{2}-\d{4}\b"
+    CREDIT_CARD_PATTERN = r"\b(?:\d{4}[-\s]?){3}\d{4}\b"
+    EMAIL_PATTERN = r"\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b"
+
+    @classmethod
+    def redact_pii(cls, text: str) -> str:
+        """
+        Anonymizes sensitive tokens before hashing or embedding.
+        """
+        if not text:
+            return ""
+        text = re.sub(cls.SSN_PATTERN, "[REDACTED_SSN]", text)
+        text = re.sub(cls.CREDIT_CARD_PATTERN, "[REDACTED_CC]", text)
+        text = re.sub(cls.EMAIL_PATTERN, "[REDACTED_EMAIL]", text)
+        return text
+
+    @staticmethod
+    def extract_system_and_user_prompts(messages: List[Dict[str, Any]]) -> Tuple[str, str, bool]:
+        """
+        Extracts concatenated system prompt and last user prompt.
+        Also returns a boolean indicating if multimodal/image content is detected.
+        """
+        system_parts = []
+        user_parts = []
+        is_multimodal = False
+        
+        for msg in messages:
+            role = msg.get("role", "")
+            content = msg.get("content", "")
+            
+            if isinstance(content, list):
+                # Multimodal format: [{type: 'text', text: '...'}, {type: 'image_url', ...}]
+                text_subparts = []
+                for part in content:
+                    if isinstance(part, dict):
+                        if part.get("type") == "text":
+                            text_subparts.append(part.get("text", ""))
+                        elif part.get("type") in ("image_url", "input_audio", "file"):
+                            is_multimodal = True
+                content_str = " ".join(text_subparts)
+            else:
+                content_str = str(content) if content is not None else ""
+                
+            if role == "system":
+                system_parts.append(content_str)
+            elif role == "user":
+                user_parts.append(content_str)
+                
+        system_prompt = "\n".join(system_parts).strip()
+        last_user_prompt = user_parts[-1] if user_parts else ""
+        return system_prompt, last_user_prompt, is_multimodal
+
+    @classmethod
+    def compute_exact_hash(cls, payload: Dict[str, Any], org_id: str = "default") -> str:
+        """
+        Computes a deterministic SHA-256 hash representing the exact request signature.
+        Includes model, messages, temperature, response_format (schema), tools, and stop sequences.
+        """
+        normalized_data = {
+            "org_id": org_id,
+            "model": payload.get("model", "").strip().lower(),
+            "messages": payload.get("messages", []),
+            "temperature": payload.get("temperature", 1.0),
+            "response_format": payload.get("response_format", None),
+            "tools": payload.get("tools", None),
+            "tool_choice": payload.get("tool_choice", None),
+            "stop": payload.get("stop", None)
+        }
+        
+        # Serialize to deterministic JSON with sorted keys
+        json_bytes = json.dumps(normalized_data, sort_keys=True, separators=(',', ':')).encode('utf-8')
+        return hashlib.sha256(json_bytes).hexdigest()
+
+    @classmethod
+    def compute_schema_hash(cls, response_format: Optional[Dict[str, Any]]) -> str:
+        """
+        Computes deterministic hash for JSON Schema structured outputs.
+        """
+        if not response_format:
+            return "no_schema"
+        raw_bytes = json.dumps(response_format, sort_keys=True, separators=(',', ':')).encode('utf-8')
+        return hashlib.sha256(raw_bytes).hexdigest()[:16]
+
+    @classmethod
+    def compute_tools_hash(cls, tools: Optional[List[Dict[str, Any]]]) -> str:
+        """
+        Computes deterministic hash for agent tool and function definitions.
+        """
+        if not tools:
+            return "no_tools"
+        raw_bytes = json.dumps(tools, sort_keys=True, separators=(',', ':')).encode('utf-8')
+        return hashlib.sha256(raw_bytes).hexdigest()[:16]

core/privacy_shield.py

@@ -0,0 +1,93 @@
+"""
+Zero-Knowledge Enterprise Privacy Shield & Reversible PII Tokenizer.
+Automatically scrubs SSNs, Credit Cards, Emails, API Keys, and PHI before sending to upstream LLMs,
+and seamlessly rehydrates original data on response delivery for HIPAA & SOC2 compliance.
+"""
+
+import re
+from typing import Dict, Tuple, List, Any
+
+# Enterprise PII Detection Regex Patterns
+PATTERNS = {
+    "SSN": re.compile(r"\b\d{3}-\d{2}-\d{4}\b"),
+    "CREDIT_CARD": re.compile(r"\b(?:4[0-9]{12}(?:[0-9]{3})?|5[1-5][0-9]{14}|3[47][0-9]{13}|6(?:011|5[0-9]{2})[0-9]{12})\b"),
+    "EMAIL": re.compile(r"\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}\b"),
+    "API_KEY": re.compile(r"\b(sk-[A-Za-z0-9]{20,}|ghp_[A-Za-z0-9]{20,}|AKIA[0-9A-Z]{16})\b"),
+    "PHONE": re.compile(r"\b(?:\+?1[-.\s]?)?\(?[2-9]\d{2}\)?[-.\s]?\d{3}[-.\s]?\d{4}\b")
+}
+
+class PrivacyShield:
+    """Reversible PII/PHI scrubbing and token rehydration engine."""
+
+    @classmethod
+    def sanitize_text(cls, text: str) -> Tuple[str, Dict[str, str], int]:
+        """
+        Replaces sensitive PII instances with deterministic tokens.
+        Returns (sanitized_text, token_map, total_redactions).
+        """
+        token_map = {}
+        counter = 1
+        sanitized = text
+
+        for pii_type, regex in PATTERNS.items():
+            matches = list(set(regex.findall(sanitized)))
+            for match in matches:
+                token = f"[REDACTED_{pii_type}_{counter}]"
+                token_map[token] = match
+                sanitized = sanitized.replace(match, token)
+                counter += 1
+
+        return sanitized, token_map, (counter - 1)
+
+    @classmethod
+    def sanitize_payload(cls, payload: Dict[str, Any]) -> Tuple[Dict[str, Any], Dict[str, str], int]:
+        """
+        Recursively sanitizes all message contents in an OpenAI / Claude payload.
+        """
+        sanitized_payload = dict(payload)
+        master_token_map = {}
+        total_scrubbed = 0
+
+        messages = sanitized_payload.get("messages", [])
+        new_messages = []
+
+        for m in messages:
+            m_copy = dict(m)
+            content = m_copy.get("content", "")
+            if isinstance(content, str):
+                s_text, t_map, count = cls.sanitize_text(content)
+                m_copy["content"] = s_text
+                master_token_map.update(t_map)
+                total_scrubbed += count
+            new_messages.append(m_copy)
+
+        sanitized_payload["messages"] = new_messages
+        return sanitized_payload, master_token_map, total_scrubbed
+
+    @classmethod
+    def rehydrate_response(cls, response_payload: Dict[str, Any], token_map: Dict[str, str]) -> Dict[str, Any]:
+        """
+        Restores original sensitive values into the assistant response text.
+        """
+        if not token_map:
+            return response_payload
+
+        resp_copy = dict(response_payload)
+        choices = resp_copy.get("choices", [])
+        for c in choices:
+            msg = c.get("message", {})
+            if "content" in msg and isinstance(msg["content"], str):
+                for token, original in token_map.items():
+                    msg["content"] = msg["content"].replace(token, original)
+
+        # Anthropic format
+        if "content" in resp_copy and isinstance(resp_copy["content"], list):
+            for block in resp_copy["content"]:
+                if isinstance(block, dict) and "text" in block:
+                    for token, original in token_map.items():
+                        block["text"] = block["text"].replace(token, original)
+
+        return resp_copy
+
+# Global Privacy Shield instance
+privacy_shield = PrivacyShield()

core/radix_tree.py

@@ -0,0 +1,115 @@
+"""
+Radix Prefix-Tree Engine for Multi-Turn AI Agent Dialogues.
+Enables conversation branching, prefix sub-tree reuse, and 1024-token ephemeral cache alignment.
+"""
+
+import hashlib
+import json
+import time
+from typing import Dict, List, Any, Optional, Tuple
+
+class RadixNode:
+    """A single conversation turn node in the Radix prefix tree."""
+    def __init__(self, node_id: str, role: str, content_hash: str, turn_index: int):
+        self.node_id = node_id
+        self.role = role
+        self.content_hash = content_hash
+        self.turn_index = turn_index
+        self.children: Dict[str, "RadixNode"] = {}  # child_content_hash -> RadixNode
+        self.cached_completion: Optional[Dict[str, Any]] = None
+        self.tool_calls: Optional[List[Dict[str, Any]]] = None
+        self.created_at = time.time()
+        self.access_count = 0
+        self.last_accessed = time.time()
+
+class RadixPrefixTree:
+    """In-memory Radix Prefix Tree for multi-turn conversations and agent loops."""
+    def __init__(self):
+        self.root = RadixNode(node_id="root", role="system", content_hash="root", turn_index=-1)
+        self.total_nodes = 1
+        self.prefix_hits = 0
+        self.exact_hits = 0
+
+    @staticmethod
+    def hash_turn(turn: Dict[str, Any]) -> str:
+        """Computes a deterministic hash of a single message turn."""
+        role = turn.get("role", "")
+        content = turn.get("content", "")
+        tool_calls = turn.get("tool_calls", None)
+        raw_repr = f"{role}:{content}:{json.dumps(tool_calls, sort_keys=True)}"
+        return hashlib.sha256(raw_repr.encode("utf-8")).hexdigest()[:16]
+
+    def match_prefix(self, messages: List[Dict[str, Any]]) -> Tuple[int, Optional[RadixNode]]:
+        """
+        Traverses the tree to find the longest matching prefix of message turns.
+        Returns (matched_turn_count, last_matched_node).
+        """
+        curr = self.root
+        matched_turns = 0
+
+        for i, turn in enumerate(messages):
+            turn_hash = self.hash_turn(turn)
+            if turn_hash in curr.children:
+                curr = curr.children[turn_hash]
+                curr.access_count += 1
+                curr.last_accessed = time.time()
+                matched_turns += 1
+            else:
+                break
+
+        if matched_turns > 0:
+            self.prefix_hits += 1
+
+        return matched_turns, (curr if curr is not self.root else None)
+
+    def insert_conversation(self, messages: List[Dict[str, Any]], completion: Dict[str, Any], tool_calls: Optional[List[Dict[str, Any]]] = None) -> RadixNode:
+        """
+        Inserts a full conversation path into the radix tree and stores the terminal completion.
+        """
+        curr = self.root
+        for i, turn in enumerate(messages):
+            turn_hash = self.hash_turn(turn)
+            if turn_hash not in curr.children:
+                new_node_id = f"node_{self.total_nodes}_{turn_hash[:8]}"
+                new_node = RadixNode(
+                    node_id=new_node_id,
+                    role=turn.get("role", "user"),
+                    content_hash=turn_hash,
+                    turn_index=i
+                )
+                curr.children[turn_hash] = new_node
+                self.total_nodes += 1
+            curr = curr.children[turn_hash]
+
+        curr.cached_completion = completion
+        curr.tool_calls = tool_calls
+        curr.access_count += 1
+        curr.last_accessed = time.time()
+        return curr
+
+    def align_ephemeral_cache_blocks(self, messages: List[Dict[str, Any]], block_size_tokens: int = 1024) -> List[Dict[str, Any]]:
+        """
+        Aligns message turns to downstream provider (Anthropic/OpenAI) 1024-token prompt caching blocks.
+        Injects Anthropic cache_control metadata on the last turn that crosses the 1024-token boundary.
+        """
+        cumulative_tokens = 0
+        aligned_messages = []
+
+        for turn in messages:
+            turn_copy = dict(turn)
+            # Estimate token count ~ words * 1.3
+            content_str = str(turn.get("content", ""))
+            est_tokens = int(len(content_str.split()) * 1.3) + 4
+            cumulative_tokens += est_tokens
+
+            if cumulative_tokens >= block_size_tokens and "cache_control" not in turn_copy:
+                # Add ephemeral cache breakpoint
+                turn_copy["cache_control"] = {"type": "ephemeral"}
+                cumulative_tokens = 0  # reset for next block
+
+            aligned_messages.append(turn_copy)
+
+        return aligned_messages
+
+# Global Radix Prefix Tree instance
+radix_tree = RadixPrefixTree()