ct-toolkit 0.3.16__tar.gz → 0.3.18__tar.gz

{ct_toolkit-0.3.16 → ct_toolkit-0.3.18}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ct-toolkit
-Version: 0.3.16
+Version: 0.3.18
 Summary: Computational Theseus Toolkit — Identity Continuity Guardrails for Agentic Systems
 Author: Hakan Damar
 License: Apache-2.0
@@ -120,9 +120,9 @@ ct-toolkit list-templates
 
 | Metric           | Status                                                  |
 | :--------------- | :------------------------------------------------------ |
-| **Tests**        | ✅ 293 passed, 3 skipped, 13 warnings (93% coverage)                       |
+| **Tests**        | ✅ 297 passed, 0 failed (100% success rate, 93% coverage)                       |
 | **Downloads**    | [![PyPI Downloads](https://static.pepy.tech/personalized-badge/ct-toolkit?period=total&units=INTERNATIONAL_SYSTEM&left_color=BLACK&right_color=MAGENTA&left_text=downloads)](https://pepy.tech/projects/ct-toolkit) |
-| **Last Phase**   | ✅ v0.3.16: Added custom judge_model and fixed Ollama L3 runner  |
+| **Last Phase**   | ✅ v0.3.18: L3 ICM graceful fallback for tool-call parsing failures  |
 | **Current Goal** | 🔶 Phase 7: Multi-Agent Synchronization (Integration)   |
 
 ### Framework & Model Support

{ct_toolkit-0.3.16 → ct_toolkit-0.3.18}/README.md

@@ -77,9 +77,9 @@ ct-toolkit list-templates
 
 | Metric           | Status                                                  |
 | :--------------- | :------------------------------------------------------ |
-| **Tests**        | ✅ 293 passed, 3 skipped, 13 warnings (93% coverage)                       |
+| **Tests**        | ✅ 297 passed, 0 failed (100% success rate, 93% coverage)                       |
 | **Downloads**    | [![PyPI Downloads](https://static.pepy.tech/personalized-badge/ct-toolkit?period=total&units=INTERNATIONAL_SYSTEM&left_color=BLACK&right_color=MAGENTA&left_text=downloads)](https://pepy.tech/projects/ct-toolkit) |
-| **Last Phase**   | ✅ v0.3.16: Added custom judge_model and fixed Ollama L3 runner  |
+| **Last Phase**   | ✅ v0.3.18: L3 ICM graceful fallback for tool-call parsing failures  |
 | **Current Goal** | 🔶 Phase 7: Multi-Agent Synchronization (Integration)   |
 
 ### Framework & Model Support

{ct_toolkit-0.3.16 → ct_toolkit-0.3.18}/ct_toolkit/__init__.py

@@ -23,7 +23,7 @@ from ct_toolkit.core.exceptions import (
     ChainIntegrityError,
 )
 
-__version__ = "0.3.16"
+__version__ = "0.3.18"
 
 __all__ = [
     "TheseusWrapper",

{ct_toolkit-0.3.16 → ct_toolkit-0.3.18}/ct_toolkit/divergence/l2_judge.py

@@ -19,6 +19,7 @@ from enum import Enum
 from typing import Any
 from pydantic import BaseModel, Field
 import instructor
+import litellm
 from jinja2 import Template
 
 from ct_toolkit.utils.logger import get_logger
@@ -103,26 +104,22 @@ class LLMJudge:
         self._provider = provider
         self._model = model or self._default_model(provider)
         
-        # Fix for Ollama: Strip /v1 from base_url if present
+        # Fix for Ollama: LiteLLM's native Ollama provider expects root port (no /v1)
+        self._api_base = None
         if provider == "ollama" and hasattr(client, "base_url"):
             base_url = str(client.base_url).rstrip("/")
             if base_url.endswith("/v1"):
-                new_base = base_url[:-3]
-                logger.debug(f"Stripping /v1 from Ollama judge base_url: {base_url} -> {new_base}")
-                try:
-                    import httpx
-                    client.base_url = httpx.URL(new_base)
-                except ImportError:
-                    client.base_url = new_base
-
-        # Patch client with instructor based on provider
-        if provider == "openai":
-            self._instructor_client = instructor.from_openai(client)
-        elif provider == "anthropic":
-            self._instructor_client = instructor.from_anthropic(client)
-        else:
-            # Fallback to standard patch for other providers (e.g. Ollama)
-            self._instructor_client = instructor.patch(client)
+                self._api_base = base_url[:-3]
+                logger.debug(f"Stripping /v1 from Ollama judge api_base: {base_url} -> {self._api_base}")
+            else:
+                self._api_base = base_url
+        elif hasattr(client, "base_url"):
+            self._api_base = str(client.base_url)
+
+        self._api_key = getattr(client, "api_key", None)
+
+        # Use instructor with litellm for unified provider support
+        self._instructor_client = instructor.from_litellm(litellm.completion)
 
     @staticmethod
     def _default_model(provider: str) -> str:
@@ -149,27 +146,51 @@ class LLMJudge:
             request_text=request_text[:1000],
         )
 
+        full_model = self._model
+        # Prevent colon-to-slash replacement if model already has ollama/ prefix 
+        # or if the provider is specifically ollama
+        is_ollama = (self._provider == "ollama" or self._model.startswith("ollama/"))
+
+        if ":" in self._model and not is_ollama:
+             full_model = self._model.replace(":", "/", 1)
+        elif self._provider not in ("openai", "unknown") and not is_ollama:
+             if not self._model.startswith(f"{self._provider}/"):
+                  full_model = f"{self._provider}/{self._model}"
+
+        # Ensure ollama always has prefix if it has a colon
+        if self._provider == "ollama" and ":" in self._model and not full_model.startswith("ollama/"):
+             full_model = f"ollama/{full_model}"
+
         try:
+            kwargs: dict[str, Any] = {
+                "model": full_model,
+                "response_model": JudgeResponse,
+                "max_retries": 2,
+            }
+
+            if self._api_base:
+                kwargs["api_base"] = self._api_base
+            if self._api_key:
+                kwargs["api_key"] = self._api_key
+
             # Call using instructor for structured data
             if self._provider == "anthropic":
-                data: JudgeResponse = self._instructor_client.messages.create(
-                    model=self._model,
-                    response_model=JudgeResponse,
-                    system=system_prompt,
+                # Anthropic LiteLLM requires system prompt in extra_body or via specific messages call
+                # But instructor.from_litellm handles standard messages list
+                data: JudgeResponse = self._instructor_client.chat.completions.create(
+                    **kwargs,
                     messages=[
+                        {"role": "system", "content": system_prompt},
                         {"role": "user", "content": user_prompt},
                     ],
-                    max_retries=2
                 )
             else:
                 data: JudgeResponse = self._instructor_client.chat.completions.create(
-                    model=self._model,
-                    response_model=JudgeResponse,
+                    **kwargs,
                     messages=[
                         {"role": "system", "content": system_prompt},
                         {"role": "user", "content": user_prompt},
                     ],
-                    max_retries=2
                 )
             
             result = JudgeResult(

{ct_toolkit-0.3.16 → ct_toolkit-0.3.18}/ct_toolkit/divergence/l3_icm.py

@@ -19,13 +19,23 @@ from dataclasses import dataclass, field
 from pathlib import Path
 from typing import Any
 
+import instructor
 import litellm
+from pydantic import BaseModel, Field
 
 from ct_toolkit.utils.logger import get_logger
 
 logger = get_logger(__name__)
 
 
+# ── Pydantic Models for Structured Responses ────────────────────────────────
+
+class ProbeResponse(BaseModel):
+    """Pydantic model for structured probe response with graceful fallback."""
+    content: str = Field(..., description="Model's response text")
+    reasoning: str = Field(default="", description="Model's reasoning if provided")
+
+
 # ── Data Models ─────────────────────────────────────────────────────────────
 
 @dataclass
@@ -221,6 +231,9 @@ class ICMRunner:
         self._include_domain = include_domain_probes
         self._project_root = project_root
         self._max_probes = max_probes
+        
+        # Use instructor with litellm for structured response validation
+        self._instructor_client = instructor.from_litellm(litellm.completion)
 
     @staticmethod
     def _default_model(provider: str) -> str:
@@ -322,35 +335,93 @@ class ICMRunner:
     # ── Model Call ──────────────────────────────────────────────────────────
 
     def _call_model(self, prompt: str) -> str:
+        """
+        Calls model with structured response validation and graceful fallback.
+        
+        L3 Graceful Fallback Strategy:
+          1. Try structured call via instructor (validates against ProbeResponse schema)
+          2. If validation fails (e.g., JSON parse error), fall back to raw completion
+          3. If raw completion fails, return safe error placeholder
+        
+        This mirrors L2 Judge's approach, preventing "parse failure = hard error" behavior.
+        """
         system = self._kernel.get_system_prompt_injection()
+        full_model = self._format_model_name()
+        
+        user_content = (
+            f"{prompt}\n\n"
+            "Please provide your reasoning step-by-step wrapped in <think></think> tags "
+            "before your final response."
+        )
+
+        kwargs = self._build_litellm_kwargs(full_model, system, user_content)
 
-        # LiteLLM format
+        # ── Attempt 1: Structured call with instructor validation ────────
+        try:
+            logger.debug(f"L3 ICM attempting structured validation for model {full_model}")
+            response: ProbeResponse = self._instructor_client.chat.completions.create(
+                **kwargs,
+                response_model=ProbeResponse,
+                max_retries=1,
+            )
+            
+            # Combine reasoning and content into full response
+            full_response = f"<think>{response.reasoning}</think>\n{response.content}" \
+                if response.reasoning else response.content
+            logger.debug(f"L3 ICM structured call succeeded")
+            return full_response
+            
+        except Exception as e:
+            logger.warning(
+                f"L3 ICM structured validation failed (likely JSON/tool-call parse error): {e}. "
+                f"Falling back to raw completion."
+            )
+            
+            # ── Attempt 2: Raw litellm.completion without structured validation ────────
+            try:
+                resp = litellm.completion(**kwargs)
+                content = self._extract_response_content(resp)
+                logger.info(f"L3 ICM raw fallback succeeded")
+                return content
+                
+            except Exception as fallback_e:
+                # ── Attempt 3: Safe error response ────────
+                logger.error(
+                    f"L3 ICM both structured and raw calls failed: "
+                    f"structured={type(e).__name__}, raw={type(fallback_e).__name__}. "
+                    f"Returning error placeholder for graceful degradation."
+                )
+                return "[ERROR: Model call failed, unable to assess response]"
+
+    def _format_model_name(self) -> str:
+        """Format model name according to provider conventions."""
         full_model = self._model
-        # Prevent colon-to-slash replacement if model already has ollama/ prefix 
-        # or if the provider is specifically ollama
         is_ollama = (self._provider == "ollama" or self._model.startswith("ollama/"))
-
+        
         if ":" in self._model and not is_ollama:
-             full_model = self._model.replace(":", "/", 1)
+            full_model = self._model.replace(":", "/", 1)
         elif self._provider not in ("openai", "unknown") and not is_ollama:
-             if not self._model.startswith(f"{self._provider}/"):
-                  full_model = f"{self._provider}/{self._model}"
-
-        # Ensure ollama always has prefix if it has a colon
+            if not self._model.startswith(f"{self._provider}/"):
+                full_model = f"{self._provider}/{self._model}"
+        
         if self._provider == "ollama" and ":" in self._model and not full_model.startswith("ollama/"):
-             full_model = f"ollama/{full_model}"
+            full_model = f"ollama/{full_model}"
+        
+        return full_model
 
+    def _build_litellm_kwargs(self, full_model: str, system: str, user_content: str) -> dict[str, Any]:
+        """Build kwargs for litellm call."""
         kwargs: dict[str, Any] = {
             "model": full_model,
             "messages": [
                 {"role": "system", "content": system},
-                {"role": "user", "content": f"{prompt}\n\nPlease provide your reasoning step-by-step wrapped in <think></think> tags before your final response."},
+                {"role": "user", "content": user_content},
             ],
             "max_tokens": 512,
             "temperature": 0.0,
         }
-
-        # Extract connection info from client if possible
+        
+        # Extract connection info from client
         if hasattr(self._client, "base_url") and self._client.base_url:
             api_base = str(self._client.base_url).rstrip("/")
             if self._provider == "ollama" and api_base.endswith("/v1"):
@@ -358,31 +429,30 @@ class ICMRunner:
                 logger.debug(f"Stripping /v1 from Ollama api_base (ICM): {api_base} -> {new_base}")
                 api_base = new_base
             kwargs["api_base"] = api_base
-            
+        
         if hasattr(self._client, "api_key") and self._client.api_key:
             kwargs["api_key"] = self._client.api_key
-            
-        try:
-            resp = litellm.completion(**kwargs)
-            
-            # Extract content from LiteLLM response
-            if hasattr(resp, "choices") and resp.choices:
-                choice = resp.choices[0]
-                if hasattr(choice, "message") and choice.message:
-                    return choice.message.content or ""
-            
-            # Fallback
-            if isinstance(resp, dict):
-                if "choices" in resp and resp["choices"]:
-                    return resp["choices"][0].get("message", {}).get("content", "")
-                if "message" in resp:
-                    return resp["message"].get("content", "")
-                return str(resp)
+        
+        return kwargs
 
+    @staticmethod
+    def _extract_response_content(resp: Any) -> str:
+        """Extract text content from litellm response object."""
+        # Handle object response with .choices attribute
+        if hasattr(resp, "choices") and resp.choices:
+            choice = resp.choices[0]
+            if hasattr(choice, "message") and choice.message:
+                return choice.message.content or ""
+        
+        # Handle dict response
+        if isinstance(resp, dict):
+            if "choices" in resp and resp["choices"]:
+                return resp["choices"][0].get("message", {}).get("content", "")
+            if "message" in resp:
+                return resp["message"].get("content", "")
             return str(resp)
-        except Exception as e:
-            logger.error(f"ICM probe litellm call failed: {e}")
-            raise
+        
+        return str(resp)
 
     # ── Probe Loading ──────────────────────────────────────────────────────────
 

{ct_toolkit-0.3.16 → ct_toolkit-0.3.18}/ct_toolkit.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ct-toolkit
-Version: 0.3.16
+Version: 0.3.18
 Summary: Computational Theseus Toolkit — Identity Continuity Guardrails for Agentic Systems
 Author: Hakan Damar
 License: Apache-2.0
@@ -120,9 +120,9 @@ ct-toolkit list-templates
 
 | Metric           | Status                                                  |
 | :--------------- | :------------------------------------------------------ |
-| **Tests**        | ✅ 293 passed, 3 skipped, 13 warnings (93% coverage)                       |
+| **Tests**        | ✅ 297 passed, 0 failed (100% success rate, 93% coverage)                       |
 | **Downloads**    | [![PyPI Downloads](https://static.pepy.tech/personalized-badge/ct-toolkit?period=total&units=INTERNATIONAL_SYSTEM&left_color=BLACK&right_color=MAGENTA&left_text=downloads)](https://pepy.tech/projects/ct-toolkit) |
-| **Last Phase**   | ✅ v0.3.16: Added custom judge_model and fixed Ollama L3 runner  |
+| **Last Phase**   | ✅ v0.3.18: L3 ICM graceful fallback for tool-call parsing failures  |
 | **Current Goal** | 🔶 Phase 7: Multi-Agent Synchronization (Integration)   |
 
 ### Framework & Model Support

{ct_toolkit-0.3.16 → ct_toolkit-0.3.18}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "ct-toolkit"
-version = "0.3.16"
+version = "0.3.18"
 description = "Computational Theseus Toolkit — Identity Continuity Guardrails for Agentic Systems"
 readme = "README.md"
 requires-python = ">=3.11"