model-router-cli 1.0.0__py3-none-any.whl

app/analytics/service.py

@@ -0,0 +1,119 @@
+from typing import Dict, Any, List, Optional
+from sqlalchemy import select, func
+from sqlalchemy.ext.asyncio import AsyncSession
+from app.storage.models import RequestRecord, FeedbackRecord, ModelRecord
+from app.config.settings import get_settings
+
+settings = get_settings()
+
+
+async def calculate_cost_savings(
+    session: AsyncSession,
+    baseline_model_id: Optional[str] = None,
+) -> Dict[str, Any]:
+    """
+    Computes accurate, non-invented cost savings:
+    Baseline Cost (if all requests were sent to baseline model) vs Actual Routed Cost.
+    """
+    base_id = baseline_model_id or settings.BASELINE_MODEL_ID
+    
+    # Fetch baseline model pricing
+    res_base = await session.execute(select(ModelRecord).filter_by(id=base_id))
+    baseline_model = res_base.scalar_one_or_none()
+    
+    base_in_rate = baseline_model.cost_per_input_token if baseline_model else 0.000005
+    base_out_rate = baseline_model.cost_per_output_token if baseline_model else 0.000015
+
+    # Fetch aggregate requests
+    res = await session.execute(
+        select(
+            func.count(RequestRecord.request_id),
+            func.sum(RequestRecord.input_tokens),
+            func.sum(RequestRecord.output_tokens),
+            func.sum(RequestRecord.estimated_cost),
+        )
+    )
+    total_reqs, total_in_tok, total_out_tok, total_routed_cost = res.one()
+
+    total_reqs = total_reqs or 0
+    total_in_tok = total_in_tok or 0
+    total_out_tok = total_out_tok or 0
+    total_routed_cost = round(total_routed_cost or 0.0, 6)
+
+    # Theoretical cost if all traffic ran on baseline
+    baseline_total_cost = round((total_in_tok * base_in_rate) + (total_out_tok * base_out_rate), 6)
+    saved_amount = max(0.0, round(baseline_total_cost - total_routed_cost, 6))
+    savings_percent = round((saved_amount / baseline_total_cost * 100.0), 2) if baseline_total_cost > 0 else 0.0
+
+    return {
+        "baseline_model": base_id,
+        "total_requests": total_reqs,
+        "total_input_tokens": total_in_tok,
+        "total_output_tokens": total_out_tok,
+        "baseline_total_cost": baseline_total_cost,
+        "routed_total_cost": total_routed_cost,
+        "cost_saved_usd": saved_amount,
+        "savings_percentage": savings_percent,
+    }
+
+
+async def get_system_analytics(session: AsyncSession) -> Dict[str, Any]:
+    """
+    Collects full aggregate metrics for the dashboard.
+    """
+    # 1. Totals & averages
+    res = await session.execute(
+        select(
+            func.count(RequestRecord.request_id),
+            func.avg(RequestRecord.total_latency_ms),
+            func.avg(RequestRecord.routing_latency_ms),
+            func.avg(RequestRecord.estimated_cost),
+            func.sum(RequestRecord.estimated_cost),
+        )
+    )
+    count, avg_lat, avg_route_lat, avg_cost, sum_cost = res.one()
+
+    # 2. Distribution by model
+    model_dist_res = await session.execute(
+        select(RequestRecord.selected_model, func.count(RequestRecord.request_id))
+        .group_by(RequestRecord.selected_model)
+    )
+    model_distribution = {m: c for m, c in model_dist_res.all()}
+
+    # 3. Distribution by task type
+    task_dist_res = await session.execute(
+        select(RequestRecord.task_type, func.count(RequestRecord.request_id))
+        .group_by(RequestRecord.task_type)
+    )
+    task_distribution = {t: c for t, c in task_dist_res.all()}
+
+    # 4. Fallback rate
+    fb_res = await session.execute(
+        select(func.count(RequestRecord.request_id)).filter(RequestRecord.fallback_used == True)
+    )
+    fallback_count = fb_res.scalar() or 0
+    fallback_rate = round((fallback_count / count * 100.0), 2) if count and count > 0 else 0.0
+
+    # 5. User Feedback aggregate
+    fb_pos_res = await session.execute(select(func.count(FeedbackRecord.id)).filter(FeedbackRecord.rating == 1))
+    fb_neg_res = await session.execute(select(func.count(FeedbackRecord.id)).filter(FeedbackRecord.rating == -1))
+    pos_feedback = fb_pos_res.scalar() or 0
+    neg_feedback = fb_neg_res.scalar() or 0
+    total_feedback = pos_feedback + neg_feedback
+    quality_score = round((pos_feedback / total_feedback * 100.0), 1) if total_feedback > 0 else None
+
+    savings = await calculate_cost_savings(session)
+
+    return {
+        "total_requests": count or 0,
+        "avg_latency_ms": round(avg_lat or 0.0, 1),
+        "avg_routing_latency_ms": round(avg_route_lat or 0.0, 1),
+        "avg_cost_usd": round(avg_cost or 0.0, 6),
+        "total_cost_usd": round(sum_cost or 0.0, 6),
+        "quality_score_percent": quality_score,
+        "fallback_rate_percent": fallback_rate,
+        "fallback_count": fallback_count,
+        "model_distribution": model_distribution,
+        "task_distribution": task_distribution,
+        "savings": savings,
+    }

app/analyzer/analyzer.py

@@ -0,0 +1,67 @@
+import json
+import re
+from typing import Optional
+from app.models.schemas import RequestAnalysis, TaskType, PriorityLevel
+from app.analyzer.heuristics import analyze_request_heuristics, estimate_tokens
+from app.providers.registry import provider_registry
+from app.config.settings import get_settings
+
+settings = get_settings()
+
+CLASSIFIER_PROMPT = """You are a real-time request classifier for an intelligent LLM model router.
+Analyze the user prompt and return ONLY a JSON object matching this schema:
+{
+  "task_type": "GENERAL_QA" | "CODING" | "DEBUGGING" | "REASONING" | "SUMMARIZATION" | "EXTRACTION" | "WRITING" | "TRANSLATION" | "ANALYSIS" | "MATH" | "LONG_CONTEXT" | "CREATIVE",
+  "complexity": 0.0 to 1.0,
+  "reasoning_required": true/false,
+  "coding_required": true/false,
+  "vision_required": true/false,
+  "latency_priority": "LOW" | "MEDIUM" | "HIGH",
+  "cost_sensitivity": "LOW" | "MEDIUM" | "HIGH",
+  "quality_requirement": "LOW" | "MEDIUM" | "HIGH"
+}
+Output valid JSON only with no markdown wrapping or additional text.
+"""
+
+
+async def analyze_request(prompt: str, analyzer_mode: Optional[str] = None) -> RequestAnalysis:
+    mode = (analyzer_mode or settings.ROUTER_ANALYZER).lower()
+
+    if mode == "llm":
+        try:
+            # Attempt to use local or configured provider for classification
+            provider = provider_registry.get_provider("ollama")
+            if provider:
+                health = await provider.check_health()
+                if health.get("status") == "CONNECTED":
+                    resp = await provider.generate(
+                        prompt=f"Classify this request:\n\n{prompt[:1000]}",
+                        model_id="qwen2.5-coder",
+                        system_prompt=CLASSIFIER_PROMPT,
+                        temperature=0.1,
+                    )
+                    clean_json = re.search(r"\{.*\}", resp.content, re.DOTALL)
+                    if clean_json:
+                        data = json.loads(clean_json.group(0))
+                        comp = float(data.get("complexity", 0.5))
+                        comp_label = PriorityLevel.HIGH if comp >= 0.75 else (PriorityLevel.MEDIUM if comp >= 0.45 else PriorityLevel.LOW)
+                        return RequestAnalysis(
+                            task_type=TaskType(data.get("task_type", "GENERAL_QA")),
+                            complexity=comp,
+                            complexity_label=comp_label,
+                            reasoning_required=bool(data.get("reasoning_required", False)),
+                            coding_required=bool(data.get("coding_required", False)),
+                            vision_required=bool(data.get("vision_required", False)),
+                            context_size=estimate_tokens(prompt),
+                            latency_priority=PriorityLevel(data.get("latency_priority", "MEDIUM")),
+                            cost_sensitivity=PriorityLevel(data.get("cost_sensitivity", "MEDIUM")),
+                            quality_requirement=PriorityLevel(data.get("quality_requirement", "MEDIUM")),
+                            keywords_detected=["llm_classified"],
+                            analyzer_used="llm",
+                        )
+        except Exception:
+            # Graceful fallback to heuristics if LLM classifier is unavailable or times out
+            pass
+
+    # Default to fast, deterministic heuristics
+    return analyze_request_heuristics(prompt)

app/analyzer/heuristics.py

@@ -0,0 +1,192 @@
+import re
+from typing import List, Tuple
+from app.models.schemas import TaskType, PriorityLevel, RequestAnalysis
+
+# Regex rules and indicator vocabularies
+DEBUG_PATTERNS = [
+    r"\b(debug|bug|stacktrace|exception|error|fix this error|traceback|segfault|syntaxerror|nullpointer)\b",
+    r"\b(crash|panicked|failed test|reproduce|root cause|undefined is not a function)\b",
+]
+
+CODING_PATTERNS = [
+    r"\b(write a python|write code|implement|function|class |def |const |async def|typescript|javascript|golang|rust|react|sql query|endpoint|api client)\b",
+    r"\b(refactor|algorithm|regex|unit test|dockerfile|kubernetes yaml|prisma|fastapi)\b",
+    r"```[a-zA-Z0-9_-]*\n",
+]
+
+REASONING_PATTERNS = [
+    r"\b(explain why|reasoning|step by step|proof|derive|why did|compare and contrast|architectural decision|tradeoffs|trade-offs|consequences of)\b",
+    r"\b(game theory|logical fallacy|syllogism|first principles|hypothesize)\b",
+]
+
+MATH_PATTERNS = [
+    r"\b(calculate|integral|derivative|matrix|eigenvalue|probability|equation|differential equation|solve for x|combinatorics|standard deviation)\b",
+    r"[\$\\\^\{\}\[\]\=]{3,}",
+]
+
+SUMMARIZATION_PATTERNS = [
+    r"\b(summarize|summary|tldr|tl;dr|key points|bullet points of this|condense|digest|brief overview)\b",
+]
+
+EXTRACTION_PATTERNS = [
+    r"\b(extract|extract json|parse this table|pull all emails|convert to json|output as csv|structured data from)\b",
+]
+
+TRANSLATION_PATTERNS = [
+    r"\b(translate to|translate into|in french|in spanish|in german|in japanese|in chinese|in hindi|translate from)\b",
+]
+
+CREATIVE_PATTERNS = [
+    r"\b(write a poem|write a story|screenplay|brainstorm catchy|creative writing|dialogue between|fiction|metaphor)\b",
+]
+
+ANALYSIS_PATTERNS = [
+    r"\b(analyze|breakdown|evaluate|critique|audit|financial report|trend analysis|pros and cons)\b",
+]
+
+
+def estimate_tokens(text: str) -> int:
+    """Fast, reliable token estimation (roughly 4 characters per token)."""
+    return max(1, len(text) // 4)
+
+
+def analyze_request_heuristics(prompt: str) -> RequestAnalysis:
+    text_lower = prompt.lower()
+    tokens = estimate_tokens(prompt)
+    
+    detected_keywords: List[str] = []
+    task_scores: dict[TaskType, float] = {t: 0.0 for t in TaskType}
+    
+    # 1. Evaluate patterns
+    for p in DEBUG_PATTERNS:
+        matches = re.findall(p, text_lower)
+        if matches:
+            task_scores[TaskType.DEBUGGING] += len(matches) * 2.5
+            detected_keywords.extend(matches)
+
+    for p in CODING_PATTERNS:
+        matches = re.findall(p, text_lower)
+        if matches:
+            task_scores[TaskType.CODING] += len(matches) * 2.0
+            detected_keywords.extend(matches)
+
+    for p in REASONING_PATTERNS:
+        matches = re.findall(p, text_lower)
+        if matches:
+            task_scores[TaskType.REASONING] += len(matches) * 2.0
+            detected_keywords.extend(matches)
+
+    for p in MATH_PATTERNS:
+        matches = re.findall(p, text_lower)
+        if matches:
+            task_scores[TaskType.MATH] += len(matches) * 2.2
+            detected_keywords.extend(matches)
+
+    for p in SUMMARIZATION_PATTERNS:
+        matches = re.findall(p, text_lower)
+        if matches:
+            task_scores[TaskType.SUMMARIZATION] += len(matches) * 2.5
+            detected_keywords.extend(matches)
+
+    for p in EXTRACTION_PATTERNS:
+        matches = re.findall(p, text_lower)
+        if matches:
+            task_scores[TaskType.EXTRACTION] += len(matches) * 2.0
+            detected_keywords.extend(matches)
+
+    for p in TRANSLATION_PATTERNS:
+        matches = re.findall(p, text_lower)
+        if matches:
+            task_scores[TaskType.TRANSLATION] += len(matches) * 3.0
+            detected_keywords.extend(matches)
+
+    for p in CREATIVE_PATTERNS:
+        matches = re.findall(p, text_lower)
+        if matches:
+            task_scores[TaskType.CREATIVE] += len(matches) * 2.0
+            detected_keywords.extend(matches)
+
+    for p in ANALYSIS_PATTERNS:
+        matches = re.findall(p, text_lower)
+        if matches:
+            task_scores[TaskType.ANALYSIS] += len(matches) * 1.8
+            detected_keywords.extend(matches)
+
+    # Long context check
+    if tokens > 4000:
+        task_scores[TaskType.LONG_CONTEXT] += 3.0
+
+    # 2. Determine Primary Task
+    best_task = TaskType.GENERAL_QA
+    max_score = 0.0
+    for task, score in task_scores.items():
+        if score > max_score:
+            max_score = score
+            best_task = task
+
+    # 3. Derive Complexity (0.0 to 1.0)
+    complexity = 0.35  # Base complexity
+    
+    # Token length factor
+    if tokens > 10000:
+        complexity += 0.30
+    elif tokens > 3000:
+        complexity += 0.20
+    elif tokens > 800:
+        complexity += 0.10
+    elif tokens < 30:
+        complexity -= 0.10
+
+    # Task inherent complexity
+    if best_task in (TaskType.DEBUGGING, TaskType.MATH, TaskType.REASONING):
+        complexity += 0.30
+    elif best_task in (TaskType.CODING, TaskType.ANALYSIS, TaskType.LONG_CONTEXT):
+        complexity += 0.20
+    elif best_task in (TaskType.TRANSLATION, TaskType.EXTRACTION, TaskType.SUMMARIZATION):
+        complexity += 0.05
+    elif best_task == TaskType.GENERAL_QA and tokens < 50:
+        complexity -= 0.15
+
+    # Multi-step keyword boosts
+    if any(k in text_lower for k in ["distributed", "async", "concurrency", "deadlock", "microservices", "architecture"]):
+        complexity += 0.25
+    if any(k in text_lower for k in ["simple", "quick", "one liner", "easy", "hello world"]):
+        complexity -= 0.20
+
+    complexity = max(0.05, min(0.99, round(complexity, 2)))
+
+    # 4. Priority and Capabilities
+    coding_required = best_task in (TaskType.CODING, TaskType.DEBUGGING) or task_scores[TaskType.CODING] > 0
+    reasoning_required = best_task in (TaskType.REASONING, TaskType.DEBUGGING, TaskType.MATH) or complexity >= 0.70
+
+    if complexity >= 0.75:
+        complexity_label = PriorityLevel.HIGH
+        quality_req = PriorityLevel.HIGH
+        latency_pri = PriorityLevel.LOW
+        cost_sens = PriorityLevel.LOW
+    elif complexity >= 0.45:
+        complexity_label = PriorityLevel.MEDIUM
+        quality_req = PriorityLevel.MEDIUM
+        latency_pri = PriorityLevel.MEDIUM
+        cost_sens = PriorityLevel.MEDIUM
+    else:
+        complexity_label = PriorityLevel.LOW
+        quality_req = PriorityLevel.LOW
+        latency_pri = PriorityLevel.HIGH
+        cost_sens = PriorityLevel.HIGH
+
+    return RequestAnalysis(
+        task_type=best_task,
+        complexity=complexity,
+        complexity_label=complexity_label,
+        reasoning_required=reasoning_required,
+        coding_required=coding_required,
+        vision_required=False,
+        tools_required=False,
+        context_size=tokens,
+        latency_priority=latency_pri,
+        cost_sensitivity=cost_sens,
+        quality_requirement=quality_req,
+        keywords_detected=list(set(detected_keywords))[:10],
+        analyzer_used="rules",
+    )