llm-proxy-cli 0.5.1__tar.gz → 0.5.3__tar.gz

{llm_proxy_cli-0.5.1 → llm_proxy_cli-0.5.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: llm-proxy-cli
-Version: 0.5.1
+Version: 0.5.3
 Summary: A lightweight CLI tool for delegating LLM tasks to expert models across multiple providers.
 Author-email: Kerem Barbaros Karnabat <kbarbaros@hotmail.com>
 Classifier: Programming Language :: Python :: 3
@@ -29,7 +29,7 @@ Dynamic: license-file
 - **Active Liveness Verification**: Pings candidates with a minimal chat request to drop fake/gated models before they crash your task.
 - **Automatic Fallbacks**: Provide a comma-separated list of models. If one fails, it instantly falls back to the next.
 - **Circuit Breaker**: Built-in health tracking and cooldowns to prevent spamming dead endpoints.
-- **Reasoning Extraction**: Automatically extracts and formats hidden `<thought>` or `reasoning` blocks (e.g., from Nemotron).
+- **Reasoning Extraction**: Automatically extracts `reasoning_content` from natively supported models (e.g., DeepSeek-R1 or Nemotron) and outputs them to stderr.
 - **Streaming Native**: Built on the official OpenAI SDK for fast and reliable streaming chunks.
 
 ## 🏗️ Architecture & Under the Hood

{llm_proxy_cli-0.5.1 → llm_proxy_cli-0.5.3}/README.md

@@ -13,7 +13,7 @@
 - **Active Liveness Verification**: Pings candidates with a minimal chat request to drop fake/gated models before they crash your task.
 - **Automatic Fallbacks**: Provide a comma-separated list of models. If one fails, it instantly falls back to the next.
 - **Circuit Breaker**: Built-in health tracking and cooldowns to prevent spamming dead endpoints.
-- **Reasoning Extraction**: Automatically extracts and formats hidden `<thought>` or `reasoning` blocks (e.g., from Nemotron).
+- **Reasoning Extraction**: Automatically extracts `reasoning_content` from natively supported models (e.g., DeepSeek-R1 or Nemotron) and outputs them to stderr.
 - **Streaming Native**: Built on the official OpenAI SDK for fast and reliable streaming chunks.
 
 ## 🏗️ Architecture & Under the Hood

{llm_proxy_cli-0.5.1 → llm_proxy_cli-0.5.3}/llm_proxy_cli.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: llm-proxy-cli
-Version: 0.5.1
+Version: 0.5.3
 Summary: A lightweight CLI tool for delegating LLM tasks to expert models across multiple providers.
 Author-email: Kerem Barbaros Karnabat <kbarbaros@hotmail.com>
 Classifier: Programming Language :: Python :: 3
@@ -29,7 +29,7 @@ Dynamic: license-file
 - **Active Liveness Verification**: Pings candidates with a minimal chat request to drop fake/gated models before they crash your task.
 - **Automatic Fallbacks**: Provide a comma-separated list of models. If one fails, it instantly falls back to the next.
 - **Circuit Breaker**: Built-in health tracking and cooldowns to prevent spamming dead endpoints.
-- **Reasoning Extraction**: Automatically extracts and formats hidden `<thought>` or `reasoning` blocks (e.g., from Nemotron).
+- **Reasoning Extraction**: Automatically extracts `reasoning_content` from natively supported models (e.g., DeepSeek-R1 or Nemotron) and outputs them to stderr.
 - **Streaming Native**: Built on the official OpenAI SDK for fast and reliable streaming chunks.
 
 ## 🏗️ Architecture & Under the Hood

{llm_proxy_cli-0.5.1 → llm_proxy_cli-0.5.3}/llm_proxy_cli.py

@@ -9,7 +9,7 @@ import logging
 from openai import OpenAI
 from filelock import FileLock, Timeout
 
-__version__ = "0.5.1"
+__version__ = "0.5.3"
 
 # Optional import for anthropic
 try:
@@ -43,14 +43,15 @@ AUTO_DISCOVERY_TIMEOUT = 5        # seconds - keep the "auto" resolve snappy
 
 def get_api_key(provider):
     keys_file = os.path.join(get_config_dir(), "keys.json")
+    keys = {}
     if os.path.exists(keys_file):
         try:
             with open(keys_file, 'r', encoding='utf-8') as f:
                 keys = json.load(f)
-            env_name = f"{provider.upper()}_API_KEY"
         except Exception:
             pass
-    return os.environ.get(f"{provider.upper()}_API_KEY") or (keys.get(env_name) if 'keys' in locals() else None)
+    env_name = f"{provider.upper()}_API_KEY"
+    return os.environ.get(env_name) or keys.get(env_name)
 
 
 PROVIDERS = {
@@ -336,8 +337,16 @@ class CircuitBreaker:
             with FileLock(self.lock_file, timeout=5):
                 circuit = self.load()
                 if model_id not in circuit:
-                    circuit[model_id] = {'failures': 0, 'cooldown_until': 0}
+                    circuit[model_id] = {'failures': 0, 'cooldown_until': 0, 'last_failure_time': 0}
+                
+                # Decay old failures after 300 seconds
+                last_fail = circuit[model_id].get('last_failure_time', 0)
+                if time.time() - last_fail > 300:
+                    circuit[model_id]['failures'] = 0
+                
                 circuit[model_id]['failures'] += 1
+                circuit[model_id]['last_failure_time'] = time.time()
+                
                 if circuit[model_id]['failures'] >= self.max_failures:
                     circuit[model_id]['cooldown_until'] = time.time() + self.cooldown_seconds
                     circuit[model_id]['failures'] = 0
@@ -417,7 +426,6 @@ def query_ai(models_list, prompt, cb: CircuitBreaker, max_retries=2, base_timeou
                     ) as stream:
                         for text in stream.text_stream:
                             if stream_out:
-                                import sys
                                 sys.stdout.write(text)
                                 sys.stdout.flush()
                             full_content += text
@@ -436,13 +444,11 @@ def query_ai(models_list, prompt, cb: CircuitBreaker, max_retries=2, base_timeou
                         if reasoning:
                             full_reasoning += reasoning
                             if stream_out:
-                                import sys
                                 sys.stderr.write(reasoning)
                                 sys.stderr.flush()
                         content = chunk.choices[0].delta.content
                         if content:
                             if stream_out:
-                                import sys
                                 sys.stdout.write(content)
                                 sys.stdout.flush()
                             full_content += content
@@ -456,6 +462,9 @@ def query_ai(models_list, prompt, cb: CircuitBreaker, max_retries=2, base_timeou
             except Exception as e:
                 error_msg = str(e).lower()
                 logger.error(f"Attempt {attempt+1} failed for {resolved_key}: {str(e)}")
+                if stream_out and (full_content or full_reasoning):
+                    sys.stderr.write(f"\n[STREAM INTERRUPTED: {str(e)} - FALLBACK TRIGGERED]\n")
+                    sys.stderr.flush()
                 status_code = getattr(e, "status_code", None)
                 if status_code in (404, 401, 403) or "404" in error_msg or "not found" in error_msg or "auth" in error_msg:
                     break
@@ -474,7 +483,7 @@ def main():
     if hasattr(sys.stdout, 'reconfigure'):
         sys.stdout.reconfigure(encoding='utf-8')
 
-    parser = argparse.ArgumentParser(description="Smart Router: A fault-tolerant CLI tool for LLM delegation.")
+    parser = argparse.ArgumentParser(description="LLM Proxy CLI: A fault-tolerant CLI tool for LLM delegation.")
     parser.add_argument("-v", "--version", action="version", version=f"LLM Proxy CLI v{__version__}")
     parser.add_argument("-m", "--models", required=True, help="Comma-separated list of provider:model fallbacks (e.g. nvidia:nemotron,groq:llama3, or groq:auto-smart / groq:auto-fast).")
     parser.add_argument("-p", "--prompt", help="The prompt text to send to the model.")
@@ -506,9 +515,8 @@ def main():
 
     cb = CircuitBreaker(args.project, args.max_failures, args.cooldown)
     try:
-        response = query_ai(args.models, prompt_text, cb, force_refresh_auto=args.refresh_models, stream_out=True)
-        if not sys.stdout.isatty():
-            print(response)
+        query_ai(args.models, prompt_text, cb, force_refresh_auto=args.refresh_models, stream_out=True)
+        print()
     except Exception as e:
         logger.error(str(e))
         sys.exit(1)

{llm_proxy_cli-0.5.1 → llm_proxy_cli-0.5.3}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "llm-proxy-cli"
-version = "0.5.1"
+version = "0.5.3"
 authors = [
   { name="Kerem Barbaros Karnabat", email="kbarbaros@hotmail.com" }
 ]

{llm_proxy_cli-0.5.1 → llm_proxy_cli-0.5.3}/tests/test_circuit.py

@@ -46,3 +46,46 @@ def test_circuit_breaker_success_reset(tmp_path):
     cb.record_success("modelC")
     circuit = cb.load()
     assert "modelC" not in circuit or circuit["modelC"]["failures"] == 0
+
+import time
+
+def test_circuit_breaker_decay():
+    cb = CircuitBreaker("decay_test", max_failures=2, cooldown_seconds=60)
+    
+    # Clean state
+    if os.path.exists(cb.circuit_file):
+        os.remove(cb.circuit_file)
+        
+    import llm_proxy_cli
+    
+    # Mock time.time to simulate passage of time
+    original_time = time.time
+    current_mock_time = original_time()
+    
+    def mock_time():
+        return current_mock_time
+        
+    llm_proxy_cli.time.time = mock_time
+    
+    try:
+        # Failure 1 at T=0
+        cb.record_failure("test:model")
+        assert cb.check_health("test:model") == True
+        
+        # Advance time by 400 seconds (past 300s decay threshold)
+        current_mock_time += 400
+        
+        # Failure 2 at T=400
+        # This should reset the counter to 0 before adding 1, so it won't trip (max_failures=2)
+        cb.record_failure("test:model")
+        assert cb.check_health("test:model") == True
+        
+        # Failure 3 immediately after (T=400)
+        # Counter becomes 2 -> trips!
+        cb.record_failure("test:model")
+        assert cb.check_health("test:model") == False
+        
+    finally:
+        llm_proxy_cli.time.time = original_time
+        if os.path.exists(cb.circuit_file):
+            os.remove(cb.circuit_file)

{llm_proxy_cli-0.5.1 → llm_proxy_cli-0.5.3}/tests/test_query_ai.py

@@ -197,3 +197,38 @@ def test_query_ai_resolves_auto_alias_before_calling_provider(monkeypatch, mock_
     # circuit breaker must key on the resolved model, never on the literal "auto-smart" alias
     mock_cb.check_health.assert_called_with("groq:resolved-model-xyz")
     mock_cb.record_success.assert_called_once_with("groq:resolved-model-xyz")
+
+
+def test_query_ai_streaming_output(capsys, monkeypatch, mock_cb):
+    """Test that stream_out=True sends reasoning to stderr and content to stdout, exactly once."""
+    monkeypatch.setitem(router.PROVIDERS, "groq", {"base_url": "dummy", "api_key": "key"})
+    
+    mock_client = MagicMock()
+    mock_chunk1 = MagicMock()
+    mock_chunk1.choices = [MagicMock()]
+    mock_chunk1.choices[0].delta.reasoning_content = "thinking..."
+    mock_chunk1.choices[0].delta.content = ""
+    
+    mock_chunk2 = MagicMock()
+    mock_chunk2.choices = [MagicMock()]
+    mock_chunk2.choices[0].delta.reasoning_content = ""
+    mock_chunk2.choices[0].delta.content = "hello world"
+    
+    mock_client.chat.completions.create.return_value = [mock_chunk1, mock_chunk2]
+    
+    factory = openai_factory({"dummy": mock_client})
+    with patch.object(router, "OpenAI", side_effect=factory):
+        response = router.query_ai("groq:llama", "hi", mock_cb, stream_out=True)
+        
+    captured = capsys.readouterr()
+    
+    # 1. Stdout must only contain "hello world" (no reasoning)
+    assert "hello world" in captured.out
+    assert "thinking..." not in captured.out
+    
+    # 2. Stderr must contain the reasoning
+    assert "thinking..." in captured.err
+    
+    # 3. The returned string still contains everything for library usage
+    assert "hello world" in response
+    assert "thinking..." in response