tactus 0.33.0__py3-none-any.whl → 0.34.0__py3-none-any.whl

tactus/stdlib/classify/fuzzy.py

@@ -0,0 +1,282 @@
+"""
+FuzzyMatchClassifier - Classification using string similarity.
+
+This classifier uses fuzzy string matching to classify text based on
+similarity to expected values. Useful for verification tasks where
+you want to check if a response matches an expected value.
+
+Supports multiple algorithms from rapidfuzz library:
+- ratio: Basic character-level similarity (default)
+- token_set_ratio: Tokenize and compare unique words (handles reordering)
+- token_sort_ratio: Sort tokens before comparing
+- partial_ratio: Best substring match
+"""
+
+import logging
+from typing import Any, Dict, List, Optional
+
+try:
+    from rapidfuzz import fuzz
+
+    HAS_RAPIDFUZZ = True
+except ImportError:
+    from difflib import SequenceMatcher
+
+    HAS_RAPIDFUZZ = False
+
+from ..core.base import BaseClassifier
+from ..core.models import ClassifierResult
+
+logger = logging.getLogger(__name__)
+
+
+def calculate_similarity(s1: str, s2: str, algorithm: str = "ratio") -> float:
+    """
+    Calculate similarity between two strings using specified algorithm.
+
+    Args:
+        s1: First string
+        s2: Second string
+        algorithm: One of "ratio", "token_set_ratio", "token_sort_ratio", "partial_ratio"
+
+    Returns:
+        Float between 0.0 (no similarity) and 1.0 (identical)
+
+    Raises:
+        ValueError: If algorithm is not supported
+
+    Note:
+        Uses rapidfuzz if available (faster), falls back to difflib for basic ratio.
+    """
+    if not s1 or not s2:
+        return 0.0
+
+    # Normalize: lowercase and strip whitespace
+    s1_norm = s1.lower().strip()
+    s2_norm = s2.lower().strip()
+
+    if HAS_RAPIDFUZZ:
+        # Use rapidfuzz (C++ backend, faster)
+        if algorithm == "token_set_ratio":
+            score = fuzz.token_set_ratio(s1_norm, s2_norm)
+        elif algorithm == "token_sort_ratio":
+            score = fuzz.token_sort_ratio(s1_norm, s2_norm)
+        elif algorithm == "partial_ratio":
+            score = fuzz.partial_ratio(s1_norm, s2_norm)
+        elif algorithm == "ratio":
+            score = fuzz.ratio(s1_norm, s2_norm)
+        else:
+            raise ValueError(
+                f"Unsupported algorithm: {algorithm}. "
+                "Choose from: ratio, token_set_ratio, token_sort_ratio, partial_ratio"
+            )
+        # Normalize from 0-100 to 0.0-1.0
+        return score / 100.0
+    else:
+        # Fallback to difflib (only supports ratio)
+        if algorithm != "ratio":
+            raise ValueError(
+                f"Algorithm '{algorithm}' requires rapidfuzz library. "
+                "Install with: pip install rapidfuzz"
+            )
+        return SequenceMatcher(None, s1_norm, s2_norm).ratio()
+
+
+class FuzzyMatchClassifier(BaseClassifier):
+    """
+    String similarity based classifier.
+
+    Compares input text against expected value(s) and returns whether
+    they match within a configurable threshold.
+
+    Two modes of operation:
+
+    1. Binary mode (single expected value):
+       Returns "Yes" if similarity >= threshold, "No" otherwise.
+
+       classifier = FuzzyMatchClassifier(
+           expected="Customer Service",
+           threshold=0.8,
+       )
+       result = classifier.classify("customer service dept")
+       # result.value = "Yes", result.confidence = 0.92
+
+    2. Multi-class mode (multiple expected values):
+       Returns the closest matching class if similarity >= threshold,
+       or "NO_MATCH" if nothing matches.
+
+       classifier = FuzzyMatchClassifier(
+           classes=["Technical Support", "Billing", "Sales"],
+           threshold=0.7,
+       )
+       result = classifier.classify("tech support")
+       # result.value = "Technical Support", result.confidence = 0.85
+
+    Example usage in Lua:
+        -- Binary: Does this match "Customer Service"?
+        result = Classify {
+            method = "fuzzy",
+            expected = "Customer Service",
+            threshold = 0.8,
+            input = agent_response
+        }
+
+        -- Multi-class: Which department?
+        result = Classify {
+            method = "fuzzy",
+            classes = {"Technical Support", "Billing", "Sales"},
+            threshold = 0.7,
+            input = department_name
+        }
+    """
+
+    def __init__(
+        self,
+        classes: Optional[List[str]] = None,
+        expected: Optional[str] = None,
+        threshold: float = 0.8,
+        algorithm: str = "ratio",
+        target_classes: Optional[List[str]] = None,
+        name: Optional[str] = None,
+        # Accept but ignore these (for factory compatibility)
+        config: Optional[Dict[str, Any]] = None,
+        **kwargs,
+    ):
+        """
+        Initialize FuzzyMatchClassifier.
+
+        Args:
+            classes: List of possible values to match against (multi-class mode)
+            expected: Single expected value (binary mode, returns Yes/No)
+            threshold: Minimum similarity score to consider a match (0.0 to 1.0)
+            algorithm: Similarity algorithm - "ratio" (default), "token_set_ratio",
+                      "token_sort_ratio", or "partial_ratio"
+            target_classes: Classes considered "positive" for precision/recall
+            name: Optional name for this classifier
+
+        Algorithm details:
+            - ratio: Character-level similarity (best for exact matches)
+            - token_set_ratio: Tokenizes and compares unique words (handles reordering)
+            - token_sort_ratio: Sorts tokens before comparing (handles reordering)
+            - partial_ratio: Best substring match (good for abbreviations)
+        """
+        # If config dict is provided, extract parameters from it
+        if config is not None:
+            classes = config.get("classes", classes)
+            expected = config.get("expected", expected)
+            threshold = config.get("threshold", threshold)
+            algorithm = config.get("algorithm", algorithm)
+            target_classes = config.get("target_classes", target_classes)
+            name = config.get("name", name)
+
+        self.threshold = threshold
+        self.algorithm = algorithm
+        self.name = name
+
+        # Determine mode: binary or multi-class
+        if expected is not None:
+            # Binary mode: Yes/No based on match to expected
+            self.mode = "binary"
+            self.expected = expected
+            self.classes = ["Yes", "No"]
+            self.target_classes = target_classes or ["Yes"]
+        elif classes is not None and len(classes) > 0:
+            # Multi-class mode: return closest matching class
+            self.mode = "multiclass"
+            self.expected = None
+            self.classes = list(classes)
+            self.target_classes = target_classes or []
+        else:
+            raise ValueError(
+                "FuzzyMatchClassifier requires either 'expected' (binary mode) "
+                "or 'classes' (multi-class mode)"
+            )
+
+        # Track statistics
+        self.total_calls = 0
+
+    def classify(self, input_text: str) -> ClassifierResult:
+        """
+        Classify the input text using fuzzy string matching.
+
+        Args:
+            input_text: The text to classify
+
+        Returns:
+            ClassifierResult with value, confidence (similarity score), explanation
+        """
+        self.total_calls += 1
+
+        if self.mode == "binary":
+            return self._classify_binary(input_text)
+        else:
+            return self._classify_multiclass(input_text)
+
+    def _classify_binary(self, input_text: str) -> ClassifierResult:
+        """
+        Binary classification: Does input match expected value?
+
+        Returns "Yes" or "No" with similarity as confidence.
+        Also returns matched_text (the expected value) for consistency.
+        """
+        similarity = calculate_similarity(input_text, self.expected, self.algorithm)
+
+        if similarity >= self.threshold:
+            return ClassifierResult(
+                value="Yes",
+                confidence=similarity,
+                matched_text=self.expected,  # Return what it matched against
+                explanation=f"Input matches expected value with {similarity:.1%} similarity using {self.algorithm} (threshold: {self.threshold:.1%})",
+            )
+        else:
+            return ClassifierResult(
+                value="No",
+                confidence=1.0 - similarity,  # Confidence in "No"
+                matched_text=None,  # No match found
+                explanation=f"Input does not match expected value. Similarity: {similarity:.1%} using {self.algorithm} (threshold: {self.threshold:.1%})",
+            )
+
+    def _classify_multiclass(self, input_text: str) -> ClassifierResult:
+        """
+        Multi-class classification: Find best matching class.
+
+        Returns the closest matching class or "NO_MATCH" if none meet threshold.
+        matched_text contains the actual matched string from the classes list.
+        """
+        best_match = None
+        best_similarity = 0.0
+
+        for cls in self.classes:
+            similarity = calculate_similarity(input_text, cls, self.algorithm)
+            if similarity > best_similarity:
+                best_similarity = similarity
+                best_match = cls
+
+        if best_similarity >= self.threshold:
+            return ClassifierResult(
+                value=best_match,
+                confidence=best_similarity,
+                matched_text=best_match,  # The actual matched class name
+                explanation=f"Best match: '{best_match}' with {best_similarity:.1%} similarity using {self.algorithm}",
+            )
+        else:
+            return ClassifierResult(
+                value="NO_MATCH",
+                confidence=1.0 - best_similarity,  # Confidence in no match
+                matched_text=None,  # No match found
+                explanation=f"No class matched above threshold using {self.algorithm}. Best was '{best_match}' at {best_similarity:.1%} (threshold: {self.threshold:.1%})",
+            )
+
+    def reset(self) -> None:
+        """Reset classifier state (no-op for fuzzy matcher)."""
+        pass
+
+    def __repr__(self) -> str:
+        if self.mode == "binary":
+            return f"FuzzyMatchClassifier(expected='{self.expected}', threshold={self.threshold}, algorithm='{self.algorithm}')"
+        else:
+            return f"FuzzyMatchClassifier(classes={self.classes}, threshold={self.threshold}, algorithm='{self.algorithm}')"
+
+
+# Also provide as FuzzyClassifier for shorter name
+FuzzyClassifier = FuzzyMatchClassifier

tactus/stdlib/classify/llm.py

@@ -0,0 +1,319 @@
+"""
+LLMClassifier - Classification using Language Models with retry logic.
+
+This classifier uses an LLM (via agent_factory) to classify text, with built-in
+retry logic that provides conversational feedback when the model returns invalid
+classifications.
+"""
+
+import logging
+import re
+from typing import Any, Callable, Dict, List, Optional
+
+from ..core.base import BaseClassifier
+from ..core.models import ClassifierResult
+
+logger = logging.getLogger(__name__)
+
+
+class LLMClassifier(BaseClassifier):
+    """
+    LLM-based classifier with automatic retry and validation.
+
+    Uses conversational feedback to help the LLM self-correct when it
+    returns invalid classifications.
+
+    Example:
+        classifier = LLMClassifier(
+            classes=["Yes", "No"],
+            prompt="Did the agent greet the customer?",
+            agent_factory=my_agent_factory,
+        )
+        result = classifier.classify("Hello, how can I help you today?")
+        # result.value = "Yes"
+        # result.confidence = 0.95
+    """
+
+    def __init__(
+        self,
+        classes: List[str],
+        prompt: str,
+        agent_factory: Callable[[Dict[str, Any]], Any],
+        target_classes: Optional[List[str]] = None,
+        max_retries: int = 3,
+        temperature: float = 0.3,
+        model: Optional[str] = None,
+        confidence_mode: str = "heuristic",
+        name: Optional[str] = None,
+    ):
+        """
+        Initialize LLMClassifier.
+
+        Args:
+            classes: List of valid classification values
+            prompt: Classification instruction/prompt
+            agent_factory: Factory function to create Agent instances
+            target_classes: Classes considered "positive" for precision/recall
+            max_retries: Maximum retry attempts on invalid output
+            temperature: LLM temperature for classification
+            model: Specific model to use (optional)
+            confidence_mode: "heuristic" or "none"
+            name: Optional name for this classifier
+        """
+        self.classes = classes
+        self.target_classes = target_classes or []
+        self.name = name
+        self.prompt = prompt
+        self.agent_factory = agent_factory
+        self.max_retries = max_retries
+        self.temperature = temperature
+        self.model = model
+        self.confidence_mode = confidence_mode
+
+        # Build classification system prompt
+        self._system_prompt = self._build_system_prompt()
+
+        # Create agent for classification
+        self._agent = self._create_agent()
+
+        # Track statistics
+        self.total_calls = 0
+        self.total_retries = 0
+
+    def _build_system_prompt(self) -> str:
+        """Build the classification system prompt."""
+        classes_str = ", ".join(f'"{c}"' for c in self.classes)
+
+        return f"""You are a classification assistant. Your task is to classify input according to the following instruction:
+
+{self.prompt}
+
+VALID CLASSIFICATIONS: {classes_str}
+
+IMPORTANT RULES:
+1. You MUST respond with EXACTLY one of the valid classifications listed above.
+2. Start your response with the classification on its own line.
+3. Then provide a brief explanation on the following lines.
+
+RESPONSE FORMAT:
+<classification>
+<explanation>
+
+Example:
+Yes
+The text clearly indicates agreement because...
+"""
+
+    def _create_agent(self) -> Any:
+        """Create the internal Agent for classification."""
+        if self.agent_factory is None:
+            raise RuntimeError("LLMClassifier requires agent_factory")
+
+        agent_config = {
+            "system_prompt": self._system_prompt,
+            "temperature": self.temperature,
+        }
+        # Optional stable name for mocking/traceability. When set, the DSL wrapper
+        # renames the internal _temp_agent_* handle so it can be mocked via:
+        #   Mocks { <name> = { message = "...", tool_calls = {...} } }
+        if self.name:
+            agent_config["name"] = self.name
+        if self.model:
+            agent_config["model"] = self.model
+
+        return self.agent_factory(agent_config)
+
+    def classify(self, input_text: str) -> ClassifierResult:
+        """
+        Classify the input text with retry logic.
+
+        Args:
+            input_text: The text to classify
+
+        Returns:
+            ClassifierResult with value, confidence, explanation
+        """
+        self.total_calls += 1
+
+        # Reset agent conversation for fresh classification
+        if hasattr(self._agent, "reset"):
+            self._agent.reset()
+
+        retry_count = 0
+        last_response = None
+
+        for attempt in range(self.max_retries + 1):
+            # Build the message for this attempt
+            if attempt == 0:
+                message = f"Please classify the following:\n\n{input_text}"
+            else:
+                # Retry with feedback
+                retry_count += 1
+                self.total_retries += 1
+                message = self._build_retry_feedback(last_response)
+                logger.debug(f"Classification retry {retry_count}: {message[:100]}...")
+
+            # Call the agent
+            try:
+                result = self._call_agent(message)
+                last_response = result.get("response") or result.get("message") or str(result)
+            except Exception as e:
+                logger.error(f"Agent call failed: {e}")
+                return ClassifierResult(
+                    value="ERROR",
+                    error=str(e),
+                    retry_count=retry_count,
+                )
+
+            # Parse the response
+            parsed = self._parse_response(last_response)
+
+            # Check if classification is valid
+            if parsed["value"] in self.classes:
+                confidence = self._extract_confidence(last_response, parsed["value"])
+                return ClassifierResult(
+                    value=parsed["value"],
+                    confidence=confidence,
+                    explanation=parsed["explanation"],
+                    retry_count=retry_count,
+                    raw_response=last_response,
+                )
+
+            logger.debug(f"Invalid classification '{parsed['value']}', retrying...")
+
+        # All retries exhausted
+        logger.warning(f"Classification failed after {self.max_retries} retries")
+        return ClassifierResult(
+            value="ERROR",
+            error=f"Max retries ({self.max_retries}) exceeded. Last response: {last_response[:200] if last_response else 'None'}",
+            retry_count=retry_count,
+            raw_response=last_response,
+        )
+
+    def _call_agent(self, message: str) -> Dict[str, Any]:
+        """Call the internal agent with a message."""
+        input_dict = {"message": message}
+        result = self._agent(input_dict)
+
+        # Convert result to dict
+        if hasattr(result, "to_dict"):
+            return result.to_dict()
+        if hasattr(result, "message"):
+            return {"response": result.message}
+        if hasattr(result, "response"):
+            return {"response": result.response}
+        if isinstance(result, dict):
+            return result
+
+        return {"response": str(result)}
+
+    def _build_retry_feedback(self, last_response: str) -> str:
+        """Build feedback message for retry."""
+        classes_str = ", ".join(f'"{c}"' for c in self.classes)
+        return f"""Your previous response was not a valid classification.
+
+Previous response: "{last_response[:200]}..."
+
+VALID CLASSIFICATIONS ARE: {classes_str}
+
+Please respond with EXACTLY one of these classifications, followed by your explanation.
+Start your response with the classification on its own line."""
+
+    def _parse_response(self, response: str) -> Dict[str, Any]:
+        """Parse classification response to extract value and explanation."""
+        if not response:
+            return {"value": None, "explanation": None}
+
+        lines = response.strip().split("\n")
+        if not lines:
+            return {"value": None, "explanation": None}
+
+        # First non-empty line should be the classification
+        first_line = lines[0].strip()
+
+        # Clean up common variations
+        # Remove markdown formatting, quotes, punctuation
+        cleaned = re.sub(r"[\*\"\'\`\:\.]", "", first_line).strip()
+
+        # Check for exact match first
+        for cls in self.classes:
+            if cleaned.lower() == cls.lower():
+                explanation = "\n".join(lines[1:]).strip() if len(lines) > 1 else None
+                return {"value": cls, "explanation": explanation}
+
+        # Check for prefix match (e.g., "Yes - the agent...")
+        for cls in self.classes:
+            if cleaned.lower().startswith(cls.lower()):
+                explanation = "\n".join(lines[1:]).strip() if len(lines) > 1 else None
+                return {"value": cls, "explanation": explanation}
+
+        # Check for the classification anywhere in first line
+        for cls in self.classes:
+            # Only match whole tokens/phrases so we don't accept accidental
+            # substrings (e.g., "Unknown" containing "No").
+            if re.search(rf"(?i)(?<![A-Za-z0-9_]){re.escape(cls)}(?![A-Za-z0-9_])", cleaned):
+                # Make sure it's not a partial match of another class
+                is_partial = False
+                for other_cls in self.classes:
+                    if other_cls != cls and cls.lower() in other_cls.lower():
+                        is_partial = True
+                        break
+                if not is_partial:
+                    explanation = "\n".join(lines[1:]).strip() if len(lines) > 1 else None
+                    return {"value": cls, "explanation": explanation}
+
+        # Could not parse - return raw first line as value
+        explanation = "\n".join(lines[1:]).strip() if len(lines) > 1 else None
+        return {"value": first_line, "explanation": explanation}
+
+    def _extract_confidence(self, response: str, classification: str) -> Optional[float]:
+        """Extract confidence from response using heuristics."""
+        if self.confidence_mode == "none":
+            return None
+
+        # Heuristic: Look for confidence indicators in the response
+        response_lower = response.lower()
+
+        # High confidence indicators
+        high_indicators = [
+            "definitely",
+            "certainly",
+            "clearly",
+            "obviously",
+            "absolutely",
+            "100%",
+            "very confident",
+        ]
+        for indicator in high_indicators:
+            if indicator in response_lower:
+                return 0.95
+
+        # Medium-high confidence
+        med_high_indicators = ["likely", "probably", "appears to be", "seems to be", "confident"]
+        for indicator in med_high_indicators:
+            if indicator in response_lower:
+                return 0.80
+
+        # Low confidence indicators
+        low_indicators = [
+            "possibly",
+            "might be",
+            "could be",
+            "not sure",
+            "uncertain",
+            "difficult to tell",
+        ]
+        for indicator in low_indicators:
+            if indicator in response_lower:
+                return 0.50
+
+        # Default confidence when no indicators found
+        return 0.75
+
+    def reset(self) -> None:
+        """Reset the classifier state (clear agent conversation)."""
+        if hasattr(self._agent, "reset"):
+            self._agent.reset()
+
+    def __repr__(self) -> str:
+        return f"LLMClassifier(classes={self.classes}, calls={self.total_calls}, retries={self.total_retries})"