agent-learning-kit 0.1.0__py3-none-any.whl

fi/evals/autoeval/prompts.py

@@ -0,0 +1,139 @@
+"""LLM prompt templates for AutoEval."""
+
+ANALYSIS_SYSTEM_PROMPT = """You are an AI evaluation expert. Your task is to analyze application descriptions and recommend appropriate evaluations and safety scanners.
+
+You must respond in valid JSON format with the following structure:
+{
+    "category": "<category>",
+    "risk_level": "<risk_level>",
+    "domain_sensitivity": "<sensitivity>",
+    "detected_features": ["<feature1>", "<feature2>"],
+    "requirements": [
+        {
+            "category": "<requirement_category>",
+            "importance": "<importance>",
+            "reason": "<explanation>",
+            "suggested_evals": ["<eval1>", "<eval2>"],
+            "suggested_scanners": ["<scanner1>", "<scanner2>"]
+        }
+    ],
+    "explanation": "<brief explanation of the analysis>"
+}
+
+## Valid Values
+
+### Categories
+- customer_support: Customer service chatbots, help desks
+- rag_system: Retrieval-augmented generation, document Q&A
+- code_assistant: Code generation, debugging, review
+- content_moderation: Content filtering, safety systems
+- agent_workflow: Autonomous agents with tool use
+- chatbot: General conversational AI
+- summarization: Text summarization
+- translation: Language translation
+- creative_writing: Content generation, copywriting
+- data_extraction: Information extraction, parsing
+- search: Search systems
+- question_answering: Q&A systems
+- unknown: Cannot determine
+
+### Risk Levels
+- low: Internal tools, development, testing (threshold: 0.6)
+- medium: General public-facing applications (threshold: 0.7)
+- high: Healthcare, finance, legal domains (threshold: 0.8)
+- critical: Safety-critical systems (threshold: 0.9)
+
+### Domain Sensitivity
+- general: No special sensitivity
+- pii_sensitive: Handles personal information
+- financial: Banking, payments, investments
+- healthcare: Medical, patient data, HIPAA
+- legal: Legal documents, contracts
+- children: Content for minors, COPPA
+- government: Government/public sector
+
+### Importance
+- required: Must have for the application to be safe/functional
+- recommended: Strongly advised but not mandatory
+- optional: Nice to have
+
+## Available Evaluations
+
+### Semantic Evaluations
+- CoherenceEval: Checks response coherence and logical flow
+
+### Agentic Evaluations
+- ActionSafetyEval: Detects unsafe agent actions
+- ReasoningQualityEval: Evaluates reasoning quality
+
+## Available Scanners
+
+### Security Scanners
+- jailbreak: Detects jailbreak and prompt manipulation attempts
+- code_injection: Detects SQL, shell, path traversal injection
+- secrets: Detects leaked API keys, passwords, credentials
+- prompt_injection: Detects prompt injection attacks
+
+### Safety Scanners
+- toxicity: Detects toxic, harmful content
+- bias: Detects biased content
+- pii: Detects personally identifiable information
+
+### Content Scanners
+- malicious_url: Detects phishing and suspicious URLs
+- invisible_chars: Detects Unicode manipulation
+- language: Validates language requirements
+- topic_restriction: Enforces topic boundaries
+
+## Guidelines
+
+1. Match the app type to the most specific category
+2. Consider the domain - healthcare, finance, legal are high-risk
+3. Recommend evaluations based on the app's primary function
+4. Recommend scanners based on safety requirements
+5. Higher risk = higher thresholds and more scanners
+6. Always include basic safety scanners for public-facing apps
+7. For RAG systems, always include faithfulness evaluations
+8. For agents, always include tool use and safety evaluations"""
+
+ANALYSIS_USER_PROMPT = """Analyze this application description and recommend evaluations and scanners:
+
+{description}
+
+Consider:
+1. What type of application is this?
+2. What are the risk factors?
+3. What domain sensitivity applies?
+4. What evaluations would ensure quality?
+5. What scanners would ensure safety?
+
+Respond with valid JSON only, no additional text."""
+
+
+CLARIFICATION_QUESTIONS_PROMPT = """Based on the application description, generate clarifying questions to better configure the evaluation pipeline.
+
+Application Description:
+{description}
+
+Current Analysis:
+- Category: {category}
+- Risk Level: {risk_level}
+- Domain Sensitivity: {domain_sensitivity}
+- Confidence: {confidence}
+
+Generate 1-3 clarifying questions that would help improve the evaluation configuration. Focus on:
+1. Deployment environment (internal vs production)
+2. Data sensitivity (PII, financial, health data)
+3. User base (general public, employees, children)
+4. Special requirements (compliance, real-time)
+
+Respond in JSON format:
+{
+    "questions": [
+        {
+            "question": "<question text>",
+            "options": ["<option1>", "<option2>", "<option3>"],
+            "impact": "<what this answer affects>"
+        }
+    ]
+}"""

fi/evals/autoeval/recommender.py

@@ -0,0 +1,242 @@
+"""Evaluation recommender for AutoEval.
+
+Maps application requirements to specific evaluations and scanners.
+"""
+
+from typing import List, Dict, Tuple, Set
+from .types import AppAnalysis, RiskLevel, DomainSensitivity
+from .config import EvalConfig, ScannerConfig
+
+
+# Mapping from requirement eval names to core metric names
+EVAL_MAPPINGS: Dict[str, str] = {
+    # Quality / Relevancy
+    "coherence": "answer_relevancy",
+    "CoherenceEval": "answer_relevancy",
+    "answer_relevancy": "answer_relevancy",
+    # RAG / Faithfulness
+    "faithfulness": "faithfulness",
+    "groundedness": "groundedness",
+    "rag_faithfulness": "rag_faithfulness",
+    "context_utilization": "context_utilization",
+    # Hallucination
+    "hallucination": "hallucination_score",
+    "factual_consistency": "factual_consistency",
+    # Agent
+    "action_safety": "action_safety",
+    "ActionSafetyEval": "action_safety",
+    "reasoning_quality": "reasoning_quality",
+    "ReasoningQualityEval": "reasoning_quality",
+    "trajectory_score": "trajectory_score",
+    "task_completion": "task_completion",
+}
+
+# Mapping from requirement scanner names to actual scanner names
+SCANNER_MAPPINGS: Dict[str, str] = {
+    # Security scanners
+    "jailbreak": "JailbreakScanner",
+    "JailbreakScanner": "JailbreakScanner",
+    "code_injection": "CodeInjectionScanner",
+    "CodeInjectionScanner": "CodeInjectionScanner",
+    "secrets": "SecretsScanner",
+    "SecretsScanner": "SecretsScanner",
+    "prompt_injection": "JailbreakScanner",  # Use jailbreak scanner for prompt injection
+    # Safety scanners (via EvalDelegateScanner)
+    "toxicity": "ToxicityScanner",
+    "ToxicityScanner": "ToxicityScanner",
+    "bias": "BiasScanner",
+    "BiasScanner": "BiasScanner",
+    "pii": "PIIScanner",
+    "PIIScanner": "PIIScanner",
+    # Content scanners
+    "malicious_url": "MaliciousURLScanner",
+    "MaliciousURLScanner": "MaliciousURLScanner",
+    "invisible_chars": "InvisibleCharScanner",
+    "InvisibleCharScanner": "InvisibleCharScanner",
+    "language": "LanguageScanner",
+    "LanguageScanner": "LanguageScanner",
+    "topic_restriction": "TopicRestrictionScanner",
+    "TopicRestrictionScanner": "TopicRestrictionScanner",
+    "regex": "RegexScanner",
+    "RegexScanner": "RegexScanner",
+}
+
+# Risk-based thresholds
+RISK_THRESHOLDS: Dict[RiskLevel, float] = {
+    RiskLevel.LOW: 0.6,
+    RiskLevel.MEDIUM: 0.7,
+    RiskLevel.HIGH: 0.8,
+    RiskLevel.CRITICAL: 0.9,
+}
+
+# Scanner actions by importance
+SCANNER_ACTIONS: Dict[str, str] = {
+    "required": "block",
+    "recommended": "flag",
+    "optional": "warn",
+}
+
+
+class EvalRecommender:
+    """
+    Maps application requirements to specific evaluations and scanners.
+
+    Example:
+        recommender = EvalRecommender()
+        evals, scanners = recommender.recommend(analysis)
+    """
+
+    def __init__(self):
+        """Initialize the recommender."""
+        self.eval_mappings = EVAL_MAPPINGS
+        self.scanner_mappings = SCANNER_MAPPINGS
+
+    def recommend(
+        self,
+        analysis: AppAnalysis,
+    ) -> Tuple[List[EvalConfig], List[ScannerConfig]]:
+        """
+        Generate evaluation and scanner recommendations.
+
+        Args:
+            analysis: Result from AppAnalyzer
+
+        Returns:
+            Tuple of (eval configs, scanner configs)
+        """
+        evals: List[EvalConfig] = []
+        scanners: List[ScannerConfig] = []
+
+        # Base threshold from risk level
+        base_threshold = RISK_THRESHOLDS.get(analysis.risk_level, 0.7)
+
+        # Track what we've added to avoid duplicates
+        added_evals: Set[str] = set()
+        added_scanners: Set[str] = set()
+
+        # Process each requirement
+        for req in analysis.requirements:
+            # Add suggested evaluations
+            for eval_name in req.suggested_evals:
+                mapped_name = self.eval_mappings.get(eval_name)
+                if mapped_name and mapped_name not in added_evals:
+                    threshold = base_threshold
+                    weight = 1.0
+
+                    # Increase threshold for required items
+                    if req.importance == "required":
+                        threshold = min(0.95, threshold + 0.05)
+                        weight = 1.5
+
+                    evals.append(
+                        EvalConfig(
+                            name=mapped_name,
+                            threshold=threshold,
+                            weight=weight,
+                        )
+                    )
+                    added_evals.add(mapped_name)
+
+            # Add suggested scanners
+            for scanner_name in req.suggested_scanners:
+                mapped_name = self.scanner_mappings.get(scanner_name)
+                if mapped_name and mapped_name not in added_scanners:
+                    action = SCANNER_ACTIONS.get(req.importance, "flag")
+
+                    scanners.append(
+                        ScannerConfig(
+                            name=mapped_name,
+                            threshold=base_threshold,
+                            action=action,
+                        )
+                    )
+                    added_scanners.add(mapped_name)
+
+        # Add domain-specific recommendations
+        domain_evals, domain_scanners = self._add_domain_recommendations(
+            analysis, base_threshold, added_evals, added_scanners
+        )
+        evals.extend(domain_evals)
+        scanners.extend(domain_scanners)
+
+        return evals, scanners
+
+    def _add_domain_recommendations(
+        self,
+        analysis: AppAnalysis,
+        base_threshold: float,
+        added_evals: Set[str],
+        added_scanners: Set[str],
+    ) -> Tuple[List[EvalConfig], List[ScannerConfig]]:
+        """Add domain-specific recommendations."""
+        evals: List[EvalConfig] = []
+        scanners: List[ScannerConfig] = []
+
+        # PII-sensitive domains always need PII scanner
+        pii_domains = {
+            DomainSensitivity.PII_SENSITIVE,
+            DomainSensitivity.HEALTHCARE,
+            DomainSensitivity.FINANCIAL,
+        }
+
+        if analysis.domain_sensitivity in pii_domains:
+            if "PIIScanner" not in added_scanners:
+                scanners.append(
+                    ScannerConfig(
+                        name="PIIScanner",
+                        threshold=base_threshold,
+                        action="redact",  # Redact PII instead of blocking
+                    )
+                )
+            if "SecretsScanner" not in added_scanners:
+                scanners.append(
+                    ScannerConfig(
+                        name="SecretsScanner",
+                        threshold=base_threshold,
+                        action="block",
+                    )
+                )
+
+        # Healthcare needs extra safety
+        if analysis.domain_sensitivity == DomainSensitivity.HEALTHCARE:
+            if "ToxicityScanner" not in added_scanners:
+                scanners.append(
+                    ScannerConfig(
+                        name="ToxicityScanner",
+                        threshold=base_threshold + 0.1,
+                        action="block",
+                    )
+                )
+
+        # Children's content needs strict safety
+        if analysis.domain_sensitivity == DomainSensitivity.CHILDREN:
+            for scanner_name in ["ToxicityScanner", "BiasScanner"]:
+                if scanner_name not in added_scanners:
+                    scanners.append(
+                        ScannerConfig(
+                            name=scanner_name,
+                            threshold=0.9,
+                            action="block",
+                        )
+                    )
+
+        # High/critical risk always needs jailbreak protection
+        if analysis.risk_level in {RiskLevel.HIGH, RiskLevel.CRITICAL}:
+            if "JailbreakScanner" not in added_scanners:
+                scanners.append(
+                    ScannerConfig(
+                        name="JailbreakScanner",
+                        threshold=base_threshold,
+                        action="block",
+                    )
+                )
+
+        return evals, scanners
+
+    def get_available_evals(self) -> List[str]:
+        """Get list of available evaluation names."""
+        return list(set(self.eval_mappings.values()))
+
+    def get_available_scanners(self) -> List[str]:
+        """Get list of available scanner names."""
+        return list(set(self.scanner_mappings.values()))