llm-proxy-cli 0.5.0__tar.gz → 0.5.2__tar.gz

{llm_proxy_cli-0.5.0 → llm_proxy_cli-0.5.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: llm-proxy-cli
-Version: 0.5.0
+Version: 0.5.2
 Summary: A lightweight CLI tool for delegating LLM tasks to expert models across multiple providers.
 Author-email: Kerem Barbaros Karnabat <kbarbaros@hotmail.com>
 Classifier: Programming Language :: Python :: 3

{llm_proxy_cli-0.5.0 → llm_proxy_cli-0.5.2}/llm_proxy_cli.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: llm-proxy-cli
-Version: 0.5.0
+Version: 0.5.2
 Summary: A lightweight CLI tool for delegating LLM tasks to expert models across multiple providers.
 Author-email: Kerem Barbaros Karnabat <kbarbaros@hotmail.com>
 Classifier: Programming Language :: Python :: 3

{llm_proxy_cli-0.5.0 → llm_proxy_cli-0.5.2}/llm_proxy_cli.py

@@ -9,7 +9,7 @@ import logging
 from openai import OpenAI
 from filelock import FileLock, Timeout
 
-__version__ = "0.5.0"
+__version__ = "0.5.2"
 
 # Optional import for anthropic
 try:
@@ -48,11 +48,9 @@ def get_api_key(provider):
             with open(keys_file, 'r', encoding='utf-8') as f:
                 keys = json.load(f)
             env_name = f"{provider.upper()}_API_KEY"
-            if keys.get(env_name):
-                return keys[env_name]
         except Exception:
             pass
-    return os.environ.get(f"{provider.upper()}_API_KEY")
+    return os.environ.get(f"{provider.upper()}_API_KEY") or (keys.get(env_name) if 'keys' in locals() else None)
 
 
 PROVIDERS = {
@@ -99,7 +97,11 @@ def _is_chat_candidate(model_id):
 def _score_model_smart(model_id):
     """Higher score = bigger / newer / more capable. Used by auto-smart."""
     name = model_id.lower()
-    # Strip dates like 2024-08-06 or context sizes like 32768
+    # Handle MoE like 8x7b -> 56b
+    def _moe(m):
+        return str(float(m.group(1)) * float(m.group(2))) + 'b'
+    name = re.sub(r'(\d+(?:\.\d+)?)\s*x\s*(\d+(?:\.\d+)?)\s*b', _moe, name)
+    # Strip dates and huge context
     name = re.sub(r'20\d{2}[-]?\d{2}[-]?\d{2}', '', name)
     name = re.sub(r'\d{4,}', '', name)
     score = 0.0
@@ -108,35 +110,36 @@ def _score_model_smart(model_id):
     elif "sonnet" in name: score += 40
     elif "haiku" in name: score += 30
 
-    # Parameter size, e.g. "8b", "70b", "405b" -> biggest wins, weighted heavily.
+    # Version extraction (handle 3-5 as 3.5 for claude/llama)
+    name_wo_size = re.sub(r'\d+(?:\.\d+)?\s*b(?!\w)', '', name)
+    ver_str = name_wo_size.replace('_', '-').replace('-r', '-')
+    ver_matches = re.findall(r'(\d+(?:[-.]\d+)*)', ver_str)
+    best_ver = 0.0
+    for v in ver_matches:
+        parts = v.replace('-', '.').split('.')
+        val = float(f"{parts[0]}.{parts[1]}") if len(parts) >= 2 else (float(parts[0]) if parts[0] else 0.0)
+        best_ver = max(best_ver, val)
+    score += best_ver * 5
+
+    # Parameter size
     size_matches = re.findall(r'(\d+(?:\.\d+)?)\s*b(?!\w)', name)
     if size_matches:
         score += max(float(s) for s in size_matches) * 10
-        name_wo_size = re.sub(r'\d+(?:\.\d+)?\s*b(?!\w)', '', name)
-    else:
-        name_wo_size = name
-
-    # Any remaining digits are treated as generation/version numbers,
-    # e.g. "llama-4", "gemini-2.5", "v3.3" -> higher wins.
-    version_matches = re.findall(r'(\d+(?:\.\d+)?)', name_wo_size)
-    if version_matches:
-        score += max(float(v) for v in version_matches) * 5
 
     for kw, bonus in (("instruct", 3), ("versatile", 3), ("reasoning", 4), ("chat", 1)):
-        if kw in name:
-            score += bonus
+        if kw in name: score += bonus
     for kw, penalty in (("mini", -2), ("lite", -2), ("tiny", -3), ("preview", -1), ("deprecated", -100)):
-        if kw in name:
-            score += penalty
+        if kw in name: score += penalty
 
     return score
 
 
 def _score_model_fast(model_id):
-    """Higher score = smaller / snappier chat model. Used by auto-fast.
-    NON_CHAT_KEYWORDS models are filtered out before this ever runs, so
-    "smallest wins" can't land on a non-chat model anymore."""
+    """Higher score = smaller / snappier chat model. Used by auto-fast."""
     name = model_id.lower()
+    def _moe(m):
+        return str(float(m.group(1)) * float(m.group(2))) + 'b'
+    name = re.sub(r'(\d+(?:\.\d+)?)\s*x\s*(\d+(?:\.\d+)?)\s*b', _moe, name)
     name = re.sub(r'20\d{2}[-]?\d{2}[-]?\d{2}', '', name)
     name = re.sub(r'\d{4,}', '', name)
     score = 0.0
@@ -144,23 +147,24 @@ def _score_model_fast(model_id):
     if "haiku" in name: score += 20
     elif "sonnet" in name: score += 10
 
+    name_wo_size = re.sub(r'\d+(?:\.\d+)?\s*b(?!\w)', '', name)
+    ver_str = name_wo_size.replace('_', '-').replace('-r', '-')
+    ver_matches = re.findall(r'(\d+(?:[-.]\d+)*)', ver_str)
+    best_ver = 0.0
+    for v in ver_matches:
+        parts = v.replace('-', '.').split('.')
+        val = float(f"{parts[0]}.{parts[1]}") if len(parts) >= 2 else (float(parts[0]) if parts[0] else 0.0)
+        best_ver = max(best_ver, val)
+    score += best_ver * 5
+
     size_matches = re.findall(r'(\d+(?:\.\d+)?)\s*b(?!\w)', name)
     if size_matches:
-        score -= max(float(s) for s in size_matches) * 10  # smaller size = higher score
-        name_wo_size = re.sub(r'\d+(?:\.\d+)?\s*b(?!\w)', '', name)
-    else:
-        name_wo_size = name
-
-    version_matches = re.findall(r'(\d+(?:\.\d+)?)', name_wo_size)
-    if version_matches:
-        score += max(float(v) for v in version_matches) * 5  # still prefer the newer generation
+        score -= max(float(s) for s in size_matches) * 10
 
     for kw, bonus in (("instant", 4), ("flash", 4), ("turbo", 4), ("mini", 3), ("lite", 3), ("small", 2), ("instruct", 1)):
-        if kw in name:
-            score += bonus
+        if kw in name: score += bonus
     for kw, penalty in (("preview", -1), ("deprecated", -100)):
-        if kw in name:
-            score += penalty
+        if kw in name: score += penalty
 
     return score
 
@@ -332,8 +336,16 @@ class CircuitBreaker:
             with FileLock(self.lock_file, timeout=5):
                 circuit = self.load()
                 if model_id not in circuit:
-                    circuit[model_id] = {'failures': 0, 'cooldown_until': 0}
+                    circuit[model_id] = {'failures': 0, 'cooldown_until': 0, 'last_failure_time': 0}
+                
+                # Decay old failures after 300 seconds
+                last_fail = circuit[model_id].get('last_failure_time', 0)
+                if time.time() - last_fail > 300:
+                    circuit[model_id]['failures'] = 0
+                
                 circuit[model_id]['failures'] += 1
+                circuit[model_id]['last_failure_time'] = time.time()
+                
                 if circuit[model_id]['failures'] >= self.max_failures:
                     circuit[model_id]['cooldown_until'] = time.time() + self.cooldown_seconds
                     circuit[model_id]['failures'] = 0
@@ -370,17 +382,19 @@ def parse_model(model_string, force_refresh_auto=False):
         if resolved:
             return provider, resolved
         logger.error(f"Auto-discovery unavailable for '{provider}' ({mode}) and no static fallback was given.")
-        return provider, model  # will fail fast in query_ai (model not found / no key)
+        return provider, None
 
     return provider, model
 
 
-def query_ai(models_list, prompt, cb: CircuitBreaker, max_retries=2, base_timeout=30, force_refresh_auto=False):
+def query_ai(models_list, prompt, cb: CircuitBreaker, max_retries=2, base_timeout=30, force_refresh_auto=False, stream_out=False):
     if isinstance(models_list, str):
         models_list = [m.strip() for m in models_list.split(',')]
 
     for current_model_str in models_list:
         provider, current_model = parse_model(current_model_str, force_refresh_auto=force_refresh_auto)
+        if not current_model:
+            continue
         # Circuit breaker tracks the *resolved* model, not the literal "auto" alias,
         # since "auto" can point at a different real model over time.
         resolved_key = f"{provider}:{current_model}"
@@ -410,6 +424,9 @@ def query_ai(models_list, prompt, cb: CircuitBreaker, max_retries=2, base_timeou
                         messages=[{"role": "user", "content": prompt}], temperature=0.7
                     ) as stream:
                         for text in stream.text_stream:
+                            if stream_out:
+                                sys.stdout.write(text)
+                                sys.stdout.flush()
                             full_content += text
                 else:
                     client = OpenAI(
@@ -423,9 +440,17 @@ def query_ai(models_list, prompt, cb: CircuitBreaker, max_retries=2, base_timeou
                     for chunk in completion:
                         if not chunk.choices: continue
                         reasoning = getattr(chunk.choices[0].delta, "reasoning_content", None)
-                        if reasoning: full_reasoning += reasoning
+                        if reasoning:
+                            full_reasoning += reasoning
+                            if stream_out:
+                                sys.stderr.write(reasoning)
+                                sys.stderr.flush()
                         content = chunk.choices[0].delta.content
-                        if content: full_content += content
+                        if content:
+                            if stream_out:
+                                sys.stdout.write(content)
+                                sys.stdout.flush()
+                            full_content += content
 
                 cb.record_success(resolved_key)
                 output = ""
@@ -436,10 +461,15 @@ def query_ai(models_list, prompt, cb: CircuitBreaker, max_retries=2, base_timeou
             except Exception as e:
                 error_msg = str(e).lower()
                 logger.error(f"Attempt {attempt+1} failed for {resolved_key}: {str(e)}")
-                cb.record_failure(resolved_key)
-                
+                if stream_out and (full_content or full_reasoning):
+                    sys.stderr.write(f"\n[STREAM INTERRUPTED: {str(e)} - FALLBACK TRIGGERED]\n")
+                    sys.stderr.flush()
                 status_code = getattr(e, "status_code", None)
-                if status_code in (429, 404, 401, 403) or "404" in error_msg or "not found" in error_msg or "auth" in error_msg:
+                if status_code in (404, 401, 403) or "404" in error_msg or "not found" in error_msg or "auth" in error_msg:
+                    break
+                if status_code != 429:
+                    cb.record_failure(resolved_key)
+                if status_code == 429:
                     break
                 
                 if attempt == max_retries - 1: break
@@ -452,8 +482,8 @@ def main():
     if hasattr(sys.stdout, 'reconfigure'):
         sys.stdout.reconfigure(encoding='utf-8')
 
-    parser = argparse.ArgumentParser(description="Smart Router: A fault-tolerant CLI tool for LLM delegation.")
-    parser.add_argument("-v", "--version", action="version", version=f"Smart Router v{__version__}")
+    parser = argparse.ArgumentParser(description="LLM Proxy CLI: A fault-tolerant CLI tool for LLM delegation.")
+    parser.add_argument("-v", "--version", action="version", version=f"LLM Proxy CLI v{__version__}")
     parser.add_argument("-m", "--models", required=True, help="Comma-separated list of provider:model fallbacks (e.g. nvidia:nemotron,groq:llama3, or groq:auto-smart / groq:auto-fast).")
     parser.add_argument("-p", "--prompt", help="The prompt text to send to the model.")
     parser.add_argument("-f", "--file", help="Path to a text file containing the prompt.")
@@ -484,8 +514,8 @@ def main():
 
     cb = CircuitBreaker(args.project, args.max_failures, args.cooldown)
     try:
-        response = query_ai(args.models, prompt_text, cb, force_refresh_auto=args.refresh_models)
-        print(response)
+        query_ai(args.models, prompt_text, cb, force_refresh_auto=args.refresh_models, stream_out=True)
+        print()
     except Exception as e:
         logger.error(str(e))
         sys.exit(1)

{llm_proxy_cli-0.5.0 → llm_proxy_cli-0.5.2}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "llm-proxy-cli"
-version = "0.5.0"
+version = "0.5.2"
 authors = [
   { name="Kerem Barbaros Karnabat", email="kbarbaros@hotmail.com" }
 ]

{llm_proxy_cli-0.5.0 → llm_proxy_cli-0.5.2}/tests/test_circuit.py

@@ -46,3 +46,46 @@ def test_circuit_breaker_success_reset(tmp_path):
     cb.record_success("modelC")
     circuit = cb.load()
     assert "modelC" not in circuit or circuit["modelC"]["failures"] == 0
+
+import time
+
+def test_circuit_breaker_decay():
+    cb = CircuitBreaker("decay_test", max_failures=2, cooldown_seconds=60)
+    
+    # Clean state
+    if os.path.exists(cb.circuit_file):
+        os.remove(cb.circuit_file)
+        
+    import llm_proxy_cli
+    
+    # Mock time.time to simulate passage of time
+    original_time = time.time
+    current_mock_time = original_time()
+    
+    def mock_time():
+        return current_mock_time
+        
+    llm_proxy_cli.time.time = mock_time
+    
+    try:
+        # Failure 1 at T=0
+        cb.record_failure("test:model")
+        assert cb.check_health("test:model") == True
+        
+        # Advance time by 400 seconds (past 300s decay threshold)
+        current_mock_time += 400
+        
+        # Failure 2 at T=400
+        # This should reset the counter to 0 before adding 1, so it won't trip (max_failures=2)
+        cb.record_failure("test:model")
+        assert cb.check_health("test:model") == True
+        
+        # Failure 3 immediately after (T=400)
+        # Counter becomes 2 -> trips!
+        cb.record_failure("test:model")
+        assert cb.check_health("test:model") == False
+        
+    finally:
+        llm_proxy_cli.time.time = original_time
+        if os.path.exists(cb.circuit_file):
+            os.remove(cb.circuit_file)

{llm_proxy_cli-0.5.0 → llm_proxy_cli-0.5.2}/tests/test_query_ai.py

@@ -117,7 +117,7 @@ def test_query_ai_stops_retrying_immediately_on_404(monkeypatch, mock_cb):
     assert result == "ok"
     # A 404 must break the retry loop after a single attempt, not consume all max_retries=3.
     assert not_found_client.chat.completions.create.call_count == 1
-    assert mock_cb.record_failure.call_count == 1
+    assert mock_cb.record_failure.call_count == 0
 
 
 def test_query_ai_skips_models_in_cooldown(monkeypatch, mock_cb):
@@ -159,10 +159,10 @@ def test_query_ai_exits_when_every_model_fails(monkeypatch, mock_cb):
     failing_client.chat.completions.create.side_effect = Exception("timeout")
     factory = openai_factory({GROQ_URL: failing_client})
     with patch.object(router, "OpenAI", side_effect=factory):
-        with pytest.raises(SystemExit) as exc_info:
+        with pytest.raises(RuntimeError) as exc_info:
             router.query_ai("groq:llama-3.3-70b-versatile", "hi", mock_cb, max_retries=2)
 
-    assert exc_info.value.code == 1
+    assert "All fallback models failed" in str(exc_info.value)
     assert failing_client.chat.completions.create.call_count == 2
 
 
@@ -197,3 +197,38 @@ def test_query_ai_resolves_auto_alias_before_calling_provider(monkeypatch, mock_
     # circuit breaker must key on the resolved model, never on the literal "auto-smart" alias
     mock_cb.check_health.assert_called_with("groq:resolved-model-xyz")
     mock_cb.record_success.assert_called_once_with("groq:resolved-model-xyz")
+
+
+def test_query_ai_streaming_output(capsys, monkeypatch, mock_cb):
+    """Test that stream_out=True sends reasoning to stderr and content to stdout, exactly once."""
+    monkeypatch.setitem(router.PROVIDERS, "groq", {"base_url": "dummy", "api_key": "key"})
+    
+    mock_client = MagicMock()
+    mock_chunk1 = MagicMock()
+    mock_chunk1.choices = [MagicMock()]
+    mock_chunk1.choices[0].delta.reasoning_content = "thinking..."
+    mock_chunk1.choices[0].delta.content = ""
+    
+    mock_chunk2 = MagicMock()
+    mock_chunk2.choices = [MagicMock()]
+    mock_chunk2.choices[0].delta.reasoning_content = ""
+    mock_chunk2.choices[0].delta.content = "hello world"
+    
+    mock_client.chat.completions.create.return_value = [mock_chunk1, mock_chunk2]
+    
+    factory = openai_factory({"dummy": mock_client})
+    with patch.object(router, "OpenAI", side_effect=factory):
+        response = router.query_ai("groq:llama", "hi", mock_cb, stream_out=True)
+        
+    captured = capsys.readouterr()
+    
+    # 1. Stdout must only contain "hello world" (no reasoning)
+    assert "hello world" in captured.out
+    assert "thinking..." not in captured.out
+    
+    # 2. Stderr must contain the reasoning
+    assert "thinking..." in captured.err
+    
+    # 3. The returned string still contains everything for library usage
+    assert "hello world" in response
+    assert "thinking..." in response