model-mesh-ai 1.0.0__tar.gz

model_mesh_ai-1.0.0/PKG-INFO

@@ -0,0 +1,8 @@
+Metadata-Version: 2.1
+Name: model-mesh-ai
+Version: 1.0.0
+Summary: Model Mesh: Unified AI Gateway SDK with Standardized Requests, Load Balancing, and Rate Limiting
+Author: Model Mesh Team
+Classifier: Programming Language :: Python :: 3
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.8

model_mesh_ai-1.0.0/README.md

@@ -0,0 +1,40 @@
+# Model Mesh - Python SDK
+
+A unified, high-performance package for interacting with Large Language Models with **Standardized Request/Response**, **Load Balancing**, and **Rate Limiting**.
+
+## Installation
+
+```bash
+cd sdk/python
+pip install -e .
+```
+
+## Quick Start
+
+```python
+import model_mesh
+
+# 1. Initialize Model Mesh
+mesh = model_mesh.Client(
+    base_url="http://localhost:8000/v1",
+    api_key="sk-modelmesh-client-key",
+    rate_limit="20/min",
+    load_balancing="round_robin"
+)
+
+# 2. Method 1: Standardized Chat (Unified Request & Response)
+response = mesh.chat(
+    model="gpt-4o",
+    messages=[{"role": "user", "content": "Explain Model Mesh!"}]
+)
+print("Assistant:", response.content)
+print("Tokens Used:", response.usage)
+
+# 3. Method 2: Inspect Load Balancing
+route = mesh.balance(model="claude-3-5-sonnet")
+print("Routed to channel:", route["selected_channel"])
+
+# 4. Method 3: Check Rate Limiting
+status = mesh.check_rate_limit(client_id="user_123")
+print("Remaining calls:", status["remaining_requests"])
+```

model_mesh_ai-1.0.0/model_mesh/__init__.py

@@ -0,0 +1,20 @@
+"""
+Model Mesh Python SDK
+A unified client and library for Model Mesh - The High-Performance AI Gateway.
+"""
+
+from .standardizer import Standardizer, ChatResponse
+from .balancer import LoadBalancer
+from .rate_limiter import RateLimiter, RateLimitExceededError
+from .client import ModelMesh, Client
+
+__version__ = "1.0.0"
+__all__ = [
+    "ModelMesh",
+    "Client",
+    "LoadBalancer",
+    "RateLimiter",
+    "Standardizer",
+    "ChatResponse",
+    "RateLimitExceededError",
+]

model_mesh_ai-1.0.0/model_mesh/balancer.py

@@ -0,0 +1,189 @@
+"""
+Model Mesh Load Balancer & Fault-Tolerance Engine
+Provides round-robin, weighted, priority failover, and circuit-breaker cooling across AI channels.
+"""
+
+import time
+import random
+from dataclasses import dataclass, field
+from typing import Any, Dict, List, Optional, Tuple, Set
+
+
+@dataclass
+class Channel:
+    name: str
+    base_url: str
+    api_keys: List[str]
+    models: List[str] = field(default_factory=lambda: ["*"])
+    weight: int = 1
+    priority: int = 1  # 1 = primary, 2 = secondary/fallback, etc.
+    key_index: int = 0
+    failure_count: int = 0
+    success_count: int = 0
+    cooldown_until: float = 0.0
+    simulated_error: Optional[str] = None  # e.g., "429: Too Many Requests" or "500: Internal Server Error"
+
+    def is_healthy(self) -> bool:
+        """Returns True if the channel is not currently cooled down and not manually faulted."""
+        return time.time() >= self.cooldown_until and self.simulated_error is None
+
+    def get_next_key(self) -> str:
+        """Round-robin through the keys of this channel."""
+        if not self.api_keys:
+            return ""
+        key = self.api_keys[self.key_index % len(self.api_keys)]
+        self.key_index = (self.key_index + 1) % len(self.api_keys)
+        return key
+
+    def trigger_cooldown(self, seconds: float = 30.0, reason: str = "Failure"):
+        """Mark channel as cooled down after an upstream failure."""
+        self.failure_count += 1
+        self.cooldown_until = time.time() + seconds
+
+    def mark_success(self):
+        """Record success and reset cooldown."""
+        self.success_count += 1
+        self.cooldown_until = 0.0
+
+    def to_dict(self) -> Dict[str, Any]:
+        now = time.time()
+        return {
+            "name": self.name,
+            "base_url": self.base_url,
+            "weight": self.weight,
+            "priority": self.priority,
+            "key_count": len(self.api_keys),
+            "healthy": self.is_healthy(),
+            "failure_count": self.failure_count,
+            "success_count": self.success_count,
+            "cooldown_remaining_sec": max(0.0, round(self.cooldown_until - now, 1)),
+            "simulated_error": self.simulated_error,
+            "supported_models": self.models,
+        }
+
+
+class LoadBalancer:
+    """
+    Intelligent router with Round-Robin, Weighted balancing, Priority Failover,
+    and Circuit-Breaker Cooldown recovery.
+    """
+
+    def __init__(self, algorithm: str = "round_robin"):
+        self.algorithm = algorithm.lower()
+        self.channels: Dict[str, Channel] = {}
+        self._round_robin_index: int = 0
+
+    def add_channel(
+        self,
+        name: str,
+        base_url: str,
+        api_keys: Optional[List[str]] = None,
+        models: Optional[List[str]] = None,
+        weight: int = 1,
+        priority: int = 1
+    ) -> Channel:
+        """Register a provider channel with custom keys, models, weight, and priority."""
+        if api_keys is None:
+            api_keys = []
+        elif isinstance(api_keys, str):
+            api_keys = [api_keys]
+
+        channel = Channel(
+            name=name,
+            base_url=base_url,
+            api_keys=api_keys,
+            models=models or ["*"],
+            weight=max(1, weight),
+            priority=priority
+        )
+        self.channels[name] = channel
+        return channel
+
+    def remove_channel(self, name: str) -> bool:
+        """Remove a channel from the balancer."""
+        if name in self.channels:
+            del self.channels[name]
+            return True
+        return False
+
+    def simulate_fault(self, name: str, error_message: str = "HTTP 429: Rate limit reached on upstream"):
+        """Inject a simulated failure into a channel to test fallback."""
+        if name in self.channels:
+            self.channels[name].simulated_error = error_message
+            self.channels[name].trigger_cooldown(45.0)
+
+    def clear_fault(self, name: str):
+        """Restore channel to healthy state."""
+        if name in self.channels:
+            self.channels[name].simulated_error = None
+            self.channels[name].cooldown_until = 0.0
+
+    def _get_eligible_channels(
+        self,
+        model: Optional[str] = None,
+        excluded: Optional[Set[str]] = None,
+        include_faulted: bool = False
+    ) -> List[Channel]:
+        """Find channels that support the requested model and are not excluded."""
+        excluded = excluded or set()
+        eligible = []
+        for ch in self.channels.values():
+            if ch.name in excluded:
+                continue
+            if not include_faulted and not ch.is_healthy():
+                continue
+            if model is None or "*" in ch.models or model in ch.models:
+                eligible.append(ch)
+        return eligible
+
+    def select_channel(
+        self,
+        model: Optional[str] = None,
+        excluded: Optional[Set[str]] = None,
+        include_faulted: bool = False
+    ) -> Tuple[Channel, str]:
+        """
+        Select best channel and key. If channels are tiered by priority,
+        highest priority (lowest number) healthy channels are picked first.
+        """
+        eligible = self._get_eligible_channels(model, excluded, include_faulted=include_faulted)
+
+        # Fallback 1: If no healthy channels with priority, try any non-excluded channel
+        if not eligible and self.channels:
+            non_excluded = [c for c in self.channels.values() if c.name not in (excluded or set())]
+            if non_excluded:
+                # pick the one with earliest cooldown expiry
+                eligible = sorted(non_excluded, key=lambda c: c.cooldown_until)[:1]
+
+        if not eligible:
+            raise RuntimeError(f"No available channels found in Model Mesh for model '{model}'.")
+
+        # Priority grouping: pick channels in the lowest available priority level
+        min_priority = min(c.priority for c in eligible)
+        tier = [c for c in eligible if c.priority == min_priority]
+
+        # 1. Weighted Selection within Tier
+        if self.algorithm == "weighted":
+            weights = [c.weight for c in tier]
+            chosen_channel = random.choices(tier, weights=weights, k=1)[0]
+        # 2. Round Robin Selection
+        else:
+            chosen_channel = tier[self._round_robin_index % len(tier)]
+            self._round_robin_index = (self._round_robin_index + 1) % len(tier)
+
+        selected_key = chosen_channel.get_next_key()
+        return chosen_channel, selected_key
+
+    def report_failure(self, channel_name: str, cooldown_seconds: float = 30.0):
+        """Record upstream error and cooldown."""
+        if channel_name in self.channels:
+            self.channels[channel_name].trigger_cooldown(cooldown_seconds)
+
+    def report_success(self, channel_name: str):
+        """Record successful response."""
+        if channel_name in self.channels:
+            self.channels[channel_name].mark_success()
+
+    def get_status(self) -> List[Dict[str, Any]]:
+        """Return snapshot of all channels."""
+        return [c.to_dict() for c in self.channels.values()]