tactus 0.32.2__py3-none-any.whl → 0.34.0__py3-none-any.whl

tactus/stdlib/core/confidence.py

@@ -0,0 +1,211 @@
+"""
+Confidence Extraction
+
+Utilities for extracting confidence scores from LLM responses.
+"""
+
+import re
+from typing import Optional
+
+
+def extract_confidence(
+    response: str,
+    mode: str = "heuristic",
+    classification: Optional[str] = None,
+) -> Optional[float]:
+    """
+    Extract confidence score from an LLM response.
+
+    Args:
+        response: The LLM response text
+        mode: Extraction mode - "heuristic", "explicit", or "none"
+        classification: The classification value (for context)
+
+    Returns:
+        Confidence score between 0.0 and 1.0, or None if extraction disabled
+
+    Modes:
+        - "heuristic": Look for confidence indicators in text (default)
+        - "explicit": Look for explicit confidence values like "Confidence: 85%"
+        - "none": Return None (confidence disabled)
+    """
+    if mode == "none":
+        return None
+
+    if mode == "explicit":
+        return _extract_explicit_confidence(response)
+
+    return _extract_heuristic_confidence(response)
+
+
+def _extract_explicit_confidence(response: str) -> Optional[float]:
+    """
+    Extract explicit confidence values from response.
+
+    Looks for patterns like:
+    - "Confidence: 85%"
+    - "confidence = 0.85"
+    - "(85% confident)"
+    """
+    patterns = [
+        r"confidence[:\s=]+(\d+)%",
+        r"confidence[:\s=]+0?\.(\d+)",
+        r"\((\d+)%\s*confident\)",
+        r"(\d+)%\s*confidence",
+    ]
+
+    response_lower = response.lower()
+
+    for pattern in patterns:
+        match = re.search(pattern, response_lower)
+        if match:
+            value = match.group(1)
+            # Convert to float between 0 and 1
+            if "." not in pattern:
+                return float(value) / 100.0
+            else:
+                return float(f"0.{value}")
+
+    # Fallback to heuristic if no explicit value found
+    return _extract_heuristic_confidence(response)
+
+
+def _extract_heuristic_confidence(response: str) -> Optional[float]:
+    """
+    Extract confidence using text heuristics.
+
+    Looks for language indicators of certainty level.
+    """
+    response_lower = response.lower()
+
+    # Very high confidence indicators (0.95)
+    very_high = [
+        "definitely",
+        "certainly",
+        "absolutely",
+        "100%",
+        "without a doubt",
+        "unquestionably",
+        "undoubtedly",
+        "clearly",
+        "obviously",
+    ]
+    for indicator in very_high:
+        if indicator in response_lower:
+            return 0.95
+
+    # High confidence indicators (0.85)
+    high = [
+        "very confident",
+        "highly likely",
+        "strongly believe",
+        "sure that",
+        "confident that",
+        "very likely",
+    ]
+    for indicator in high:
+        if indicator in response_lower:
+            return 0.85
+
+    # Medium-high confidence indicators (0.75)
+    med_high = [
+        "likely",
+        "probably",
+        "appears to be",
+        "seems to be",
+        "based on",
+        "indicates",
+        "suggests",
+    ]
+    for indicator in med_high:
+        if indicator in response_lower:
+            return 0.75
+
+    # Medium confidence indicators (0.60)
+    medium = [
+        "may be",
+        "might be",
+        "could be",
+        "possibly",
+        "perhaps",
+        "somewhat",
+    ]
+    for indicator in medium:
+        if indicator in response_lower:
+            return 0.60
+
+    # Low confidence indicators (0.45)
+    low = [
+        "not entirely sure",
+        "uncertain",
+        "difficult to determine",
+        "hard to tell",
+        "ambiguous",
+        "unclear",
+    ]
+    for indicator in low:
+        if indicator in response_lower:
+            return 0.45
+
+    # Very low confidence indicators (0.30)
+    very_low = [
+        "very uncertain",
+        "cannot determine",
+        "impossible to tell",
+        "no way to know",
+        "purely guessing",
+    ]
+    for indicator in very_low:
+        if indicator in response_lower:
+            return 0.30
+
+    # Default confidence when no indicators found
+    return 0.70
+
+
+# Confidence level mappings for string labels
+CONFIDENCE_LABELS = {
+    "very_high": 0.95,
+    "high": 0.85,
+    "medium_high": 0.75,
+    "medium": 0.60,
+    "low": 0.45,
+    "very_low": 0.30,
+}
+
+
+def confidence_to_label(confidence: float) -> str:
+    """
+    Convert numeric confidence to a label.
+
+    Args:
+        confidence: Confidence value between 0.0 and 1.0
+
+    Returns:
+        Label string: "very_high", "high", "medium_high", "medium", "low", or "very_low"
+    """
+    if confidence >= 0.90:
+        return "very_high"
+    elif confidence >= 0.80:
+        return "high"
+    elif confidence >= 0.70:
+        return "medium_high"
+    elif confidence >= 0.55:
+        return "medium"
+    elif confidence >= 0.40:
+        return "low"
+    else:
+        return "very_low"
+
+
+def label_to_confidence(label: str) -> float:
+    """
+    Convert a label to numeric confidence.
+
+    Args:
+        label: Confidence label
+
+    Returns:
+        Confidence value between 0.0 and 1.0
+    """
+    return CONFIDENCE_LABELS.get(label.lower(), 0.70)

tactus/stdlib/core/models.py

@@ -0,0 +1,161 @@
+"""
+Pydantic models for stdlib result types.
+
+These models provide:
+- Type safety and validation
+- Consistent result structures across all classifiers/extractors
+- Easy serialization for Lua interop
+"""
+
+from typing import Any, Dict, List, Optional
+from pydantic import BaseModel, Field
+
+
+class ClassifierResult(BaseModel):
+    """
+    Result from any classifier (LLM, fuzzy match, etc.).
+
+    All classifiers return this same structure, enabling polymorphism.
+    """
+
+    value: str = Field(..., description="The classification result")
+    confidence: Optional[float] = Field(
+        None,
+        ge=0.0,
+        le=1.0,
+        description="Confidence score between 0.0 and 1.0",
+    )
+    explanation: Optional[str] = Field(
+        None, description="Reasoning or explanation for the classification"
+    )
+    matched_text: Optional[str] = Field(
+        None, description="The actual text that was matched (for fuzzy matching)"
+    )
+    retry_count: int = Field(0, ge=0, description="Number of retries needed to get valid result")
+    raw_response: Optional[str] = Field(None, description="Raw response from LLM (if applicable)")
+    error: Optional[str] = Field(None, description="Error message if classification failed")
+
+    def to_lua_dict(self) -> Dict[str, Any]:
+        """Convert to dict suitable for Lua interop."""
+        return {
+            "value": self.value,
+            "confidence": self.confidence,
+            "explanation": self.explanation,
+            "matched_text": self.matched_text,
+            "retry_count": self.retry_count,
+            "raw_response": self.raw_response,
+            "error": self.error,
+        }
+
+    def to_dict(self) -> Dict[str, Any]:
+        """Convert to dict (convenience alias for to_lua_dict)."""
+        return self.to_lua_dict()
+
+    @property
+    def is_error(self) -> bool:
+        """Check if this result represents an error."""
+        return self.error is not None or self.value == "ERROR"
+
+
+class ExtractorResult(BaseModel):
+    """
+    Result from any extractor (LLM, schema-based, etc.).
+
+    Contains extracted fields plus validation information.
+    """
+
+    fields: Dict[str, Any] = Field(default_factory=dict, description="Extracted field values")
+    validation_errors: List[str] = Field(
+        default_factory=list, description="Validation errors for extracted fields"
+    )
+    retry_count: int = Field(0, ge=0, description="Number of retries needed to get valid result")
+    raw_response: Optional[str] = Field(None, description="Raw response from LLM (if applicable)")
+    error: Optional[str] = Field(None, description="Error message if extraction failed")
+
+    def to_lua_dict(self) -> Dict[str, Any]:
+        """Convert to dict suitable for Lua interop."""
+        result = dict(self.fields)  # Flatten fields to top level
+        result["_validation_errors"] = self.validation_errors
+        result["_retry_count"] = self.retry_count
+        result["_error"] = self.error
+        return result
+
+    def to_dict(self) -> Dict[str, Any]:
+        """Convert to dict (convenience alias for to_lua_dict)."""
+        return self.to_lua_dict()
+
+    @property
+    def is_valid(self) -> bool:
+        """Check if extraction was valid (no errors)."""
+        return len(self.validation_errors) == 0 and self.error is None
+
+
+class ClassifierConfig(BaseModel):
+    """
+    Configuration for a classifier.
+
+    Used to validate and document classifier options.
+    """
+
+    classes: List[str] = Field(..., min_length=2, description="Valid classification values")
+    target_classes: List[str] = Field(
+        default_factory=list,
+        description="Target classes for precision/recall metrics (subset of classes)",
+    )
+    prompt: Optional[str] = Field(None, description="Classification instruction/prompt")
+    max_retries: int = Field(3, ge=0, description="Maximum retry attempts")
+    temperature: float = Field(0.3, ge=0.0, le=2.0, description="LLM temperature")
+    model: Optional[str] = Field(None, description="Model to use (optional)")
+    confidence_mode: str = Field(
+        "heuristic",
+        description="Confidence extraction mode: 'heuristic', 'logprobs', or 'none'",
+    )
+    parse_direction: str = Field(
+        "start",
+        description="Where to look for classification: 'start', 'end', or 'any'",
+    )
+    method: str = Field("llm", description="Classification method: 'llm' or 'fuzzy'")
+
+    # Fuzzy match specific
+    expected: Optional[str] = Field(None, description="Expected value for fuzzy matching")
+    threshold: float = Field(
+        0.8, ge=0.0, le=1.0, description="Similarity threshold for fuzzy matching"
+    )
+
+
+class ExtractorConfig(BaseModel):
+    """
+    Configuration for an extractor.
+
+    Used to validate and document extractor options.
+    """
+
+    fields: Dict[str, str] = Field(
+        ..., description="Fields to extract with their types (name -> type)"
+    )
+    prompt: Optional[str] = Field(None, description="Extraction instruction/prompt")
+    max_retries: int = Field(3, ge=0, description="Maximum retry attempts")
+    temperature: float = Field(0.3, ge=0.0, le=2.0, description="LLM temperature")
+    model: Optional[str] = Field(None, description="Model to use (optional)")
+    strict: bool = Field(
+        True, description="Whether to require all fields (strict) or allow missing"
+    )
+    method: str = Field("llm", description="Extraction method: 'llm' or 'schema'")
+
+
+class EvaluationResult(BaseModel):
+    """
+    Result from evaluating a classifier/extractor on test data.
+
+    Contains metrics like accuracy, precision, recall, F1.
+    """
+
+    accuracy: float = Field(..., ge=0.0, le=1.0)
+    precision: Optional[float] = Field(None, ge=0.0, le=1.0)
+    recall: Optional[float] = Field(None, ge=0.0, le=1.0)
+    f1: Optional[float] = Field(None, ge=0.0, le=1.0)
+    confusion_matrix: Optional[Dict[str, Dict[str, int]]] = None
+    total_samples: int = Field(..., ge=0)
+    total_retries: int = Field(0, ge=0)
+    mean_confidence: Optional[float] = Field(None, ge=0.0, le=1.0)
+    errors: List[str] = Field(default_factory=list)

tactus/stdlib/core/retry.py

@@ -0,0 +1,171 @@
+"""
+Retry with Conversational Feedback
+
+Provides intelligent retry logic that preserves conversation history
+and gives the LLM feedback about previous attempts.
+"""
+
+import logging
+from typing import Any, Callable, Dict, List, Optional, TypeVar
+
+logger = logging.getLogger(__name__)
+
+T = TypeVar("T")
+
+
+class RetryWithFeedback:
+    """
+    Retry logic with conversational feedback.
+
+    Unlike simple retry, this approach:
+    1. Preserves conversation history across attempts
+    2. Gives the LLM feedback about why the previous attempt failed
+    3. Enables "self-healing" where the LLM learns from mistakes
+
+    This is the core pattern used by Plexus LangGraphScore nodes
+    for reliable classification.
+    """
+
+    def __init__(
+        self,
+        max_retries: int = 3,
+        on_retry: Optional[Callable[[int, str, str], None]] = None,
+    ):
+        """
+        Initialize retry handler.
+
+        Args:
+            max_retries: Maximum number of retry attempts
+            on_retry: Optional callback called on each retry (attempt, error, feedback)
+        """
+        self.max_retries = max_retries
+        self.on_retry = on_retry
+
+    def execute(
+        self,
+        call_fn: Callable[[str], str],
+        initial_message: str,
+        validate_fn: Callable[[str], bool],
+        build_feedback_fn: Callable[[str], str],
+    ) -> Dict[str, Any]:
+        """
+        Execute with retry logic.
+
+        Args:
+            call_fn: Function to call the LLM (takes message, returns response)
+            initial_message: Initial message to send
+            validate_fn: Function to validate response (returns True if valid)
+            build_feedback_fn: Function to build feedback message from failed response
+
+        Returns:
+            Dict with:
+                - response: The successful response (or last failed response)
+                - success: Whether validation passed
+                - retry_count: Number of retries performed
+                - history: List of all (message, response) pairs
+        """
+        history: List[Dict[str, str]] = []
+        retry_count = 0
+        current_message = initial_message
+        last_response = None
+
+        for attempt in range(self.max_retries + 1):
+            # Call the LLM
+            response = call_fn(current_message)
+            last_response = response
+
+            # Record in history
+            history.append({"message": current_message, "response": response})
+
+            # Validate the response
+            if validate_fn(response):
+                logger.debug(f"Valid response on attempt {attempt + 1}")
+                return {
+                    "response": response,
+                    "success": True,
+                    "retry_count": retry_count,
+                    "history": history,
+                }
+
+            # Response invalid - prepare for retry
+            if attempt < self.max_retries:
+                retry_count += 1
+                feedback = build_feedback_fn(response)
+                current_message = feedback
+
+                logger.debug(f"Retry {retry_count}: {feedback[:100]}...")
+
+                if self.on_retry:
+                    self.on_retry(retry_count, response, feedback)
+
+        # All retries exhausted
+        logger.warning(f"Validation failed after {self.max_retries} retries")
+        return {
+            "response": last_response,
+            "success": False,
+            "retry_count": retry_count,
+            "history": history,
+        }
+
+
+def create_classification_validator(valid_classes: List[str]) -> Callable[[str], bool]:
+    """
+    Create a validator function for classification responses.
+
+    Args:
+        valid_classes: List of valid classification values
+
+    Returns:
+        Validator function that returns True if response contains valid class
+    """
+    valid_lower = {c.lower() for c in valid_classes}
+
+    def validator(response: str) -> bool:
+        if not response:
+            return False
+
+        # Check first line for classification
+        first_line = response.strip().split("\n")[0].strip().lower()
+
+        # Remove common formatting
+        import re
+
+        cleaned = re.sub(r"[\*\"\'\`\:\.]", "", first_line).strip()
+
+        # Check for exact match
+        if cleaned in valid_lower:
+            return True
+
+        # Check for prefix match
+        for cls in valid_lower:
+            if cleaned.startswith(cls):
+                return True
+
+        return False
+
+    return validator
+
+
+def create_classification_feedback(valid_classes: List[str]) -> Callable[[str], str]:
+    """
+    Create a feedback builder for classification retries.
+
+    Args:
+        valid_classes: List of valid classification values
+
+    Returns:
+        Function that builds feedback message from failed response
+    """
+    classes_str = ", ".join(f'"{c}"' for c in valid_classes)
+
+    def build_feedback(response: str) -> str:
+        return f"""Your previous response was not a valid classification.
+
+Your response: "{response[:200]}..."
+
+VALID CLASSIFICATIONS ARE: {classes_str}
+
+Please respond with EXACTLY one of these classifications on the first line, followed by your explanation.
+Do not include any other text on the first line."""
+
+    return build_feedback