agent-os-kernel 1.1.0__py3-none-any.whl → 1.2.0__py3-none-any.whl

modules/scak/tests/test_reference_implementations.py

@@ -0,0 +1,240 @@
+"""
+Unit tests for the reference implementations.
+
+These tests validate the simplified reference implementations:
+- auditor.py: Soft failure detection
+- teacher.py: Shadow teacher diagnosis
+- memory_manager.py: Lesson lifecycle management
+"""
+
+import unittest
+import asyncio
+from datetime import datetime
+
+from agent_kernel.auditor import CompletenessAuditor as SimpleAuditor
+from agent_kernel.teacher import diagnose_failure
+from agent_kernel.memory_manager import MemoryManager, LessonType
+
+
+class TestSimpleAuditor(unittest.TestCase):
+    """Tests for the simplified auditor reference implementation."""
+    
+    def setUp(self):
+        self.auditor = SimpleAuditor()
+    
+    def test_detects_verbal_resignation(self):
+        """Test detection of lazy verbal signals."""
+        agent_response = "I cannot find the data you requested."
+        tool_output = None
+        
+        needs_intervention = self.auditor.audit_response(agent_response, tool_output)
+        self.assertTrue(needs_intervention)
+    
+    def test_detects_no_data_found(self):
+        """Test detection of 'no data found' signal."""
+        agent_response = "No data found in the system."
+        tool_output = None
+        
+        needs_intervention = self.auditor.audit_response(agent_response, tool_output)
+        self.assertTrue(needs_intervention)
+    
+    def test_detects_empty_results(self):
+        """Test detection of empty tool output."""
+        agent_response = "Here are the results:"
+        tool_output = "[]"
+        
+        needs_intervention = self.auditor.audit_response(agent_response, tool_output)
+        self.assertTrue(needs_intervention)
+    
+    def test_accepts_valid_response(self):
+        """Test that valid responses are not flagged."""
+        agent_response = "Found 42 records matching your query."
+        tool_output = '[{"id": 1, "name": "test"}]'
+        
+        needs_intervention = self.auditor.audit_response(agent_response, tool_output)
+        self.assertFalse(needs_intervention)
+    
+    def test_all_lazy_signals(self):
+        """Test all lazy signal patterns."""
+        test_cases = [
+            "I'm sorry, I cannot help with that.",
+            "Unable to access the resource.",
+            "The context does not contain this information.",
+            "No data found for your request."
+        ]
+        
+        for response in test_cases:
+            with self.subTest(response=response):
+                needs_intervention = self.auditor.audit_response(response, None)
+                self.assertTrue(needs_intervention, f"Should detect: {response}")
+
+
+class TestTeacher(unittest.TestCase):
+    """Tests for the shadow teacher reference implementation."""
+    
+    def test_diagnose_failure_basic(self):
+        """Test basic diagnosis functionality."""
+        prompt = "Find logs for error 500"
+        failed_response = "No logs found."
+        tool_trace = "search_logs(error_code='500', time_range='recent')"
+        
+        # Run async function
+        diagnosis = asyncio.run(diagnose_failure(prompt, failed_response, tool_trace))
+        
+        self.assertIsInstance(diagnosis, dict)
+        self.assertIn("cause", diagnosis)
+        self.assertIn("lesson_patch", diagnosis)
+    
+    def test_diagnosis_structure(self):
+        """Test that diagnosis has proper structure."""
+        prompt = "Delete recent user records"
+        failed_response = "Cannot delete records."
+        tool_trace = "execute_sql(query='DELETE FROM recent_users')"
+        
+        diagnosis = asyncio.run(diagnose_failure(prompt, failed_response, tool_trace))
+        
+        # Verify structure
+        self.assertIsInstance(diagnosis["cause"], str)
+        self.assertIsInstance(diagnosis["lesson_patch"], str)
+        self.assertGreater(len(diagnosis["cause"]), 0)
+        self.assertGreater(len(diagnosis["lesson_patch"]), 0)
+
+
+class TestMemoryManager(unittest.TestCase):
+    """Tests for the memory manager reference implementation."""
+    
+    def setUp(self):
+        self.manager = MemoryManager()
+    
+    def test_add_syntax_lesson(self):
+        """Test adding a syntax lesson."""
+        self.manager.add_lesson(
+            "Always output JSON format",
+            LessonType.SYNTAX
+        )
+        
+        lessons = self.manager.get_lessons_by_type(LessonType.SYNTAX)
+        self.assertEqual(len(lessons), 1)
+        self.assertEqual(lessons[0]["text"], "Always output JSON format")
+    
+    def test_add_business_lesson(self):
+        """Test adding a business lesson."""
+        self.manager.add_lesson(
+            "Fiscal year starts in October",
+            LessonType.BUSINESS
+        )
+        
+        lessons = self.manager.get_lessons_by_type(LessonType.BUSINESS)
+        self.assertEqual(len(lessons), 1)
+        self.assertEqual(lessons[0]["text"], "Fiscal year starts in October")
+    
+    def test_add_one_off_lesson(self):
+        """Test adding a one-off lesson."""
+        self.manager.add_lesson(
+            "Server maintenance on 2024-01-15",
+            LessonType.ONE_OFF
+        )
+        
+        lessons = self.manager.get_lessons_by_type(LessonType.ONE_OFF)
+        self.assertEqual(len(lessons), 1)
+    
+    def test_upgrade_purge_removes_syntax(self):
+        """Test that upgrade purge removes SYNTAX lessons."""
+        # Add various lesson types
+        self.manager.add_lesson("Output JSON", LessonType.SYNTAX)
+        self.manager.add_lesson("Use UUIDs", LessonType.SYNTAX)
+        self.manager.add_lesson("Fiscal year Oct", LessonType.BUSINESS)
+        self.manager.add_lesson("Project Alpha deprecated", LessonType.BUSINESS)
+        
+        # Verify initial state
+        self.assertEqual(len(self.manager.vector_store), 4)
+        
+        # Run upgrade purge
+        result = self.manager.run_upgrade_purge("gpt-5")
+        
+        # Verify SYNTAX lessons were purged
+        self.assertEqual(result["purged_count"], 2)
+        self.assertEqual(result["retained_count"], 2)
+        self.assertEqual(result["new_model_version"], "gpt-5")
+        
+        # Verify only BUSINESS lessons remain
+        remaining_lessons = self.manager.vector_store
+        self.assertEqual(len(remaining_lessons), 2)
+        for lesson in remaining_lessons:
+            self.assertEqual(lesson["type"], LessonType.BUSINESS)
+    
+    def test_upgrade_purge_preserves_business(self):
+        """Test that business lessons are preserved on upgrade."""
+        self.manager.add_lesson("Fiscal year starts Oct", LessonType.BUSINESS)
+        self.manager.add_lesson("Medical advice prohibited", LessonType.BUSINESS)
+        
+        result = self.manager.run_upgrade_purge("gpt-5")
+        
+        # No lessons should be purged
+        self.assertEqual(result["purged_count"], 0)
+        self.assertEqual(result["retained_count"], 2)
+    
+    def test_get_lesson_count(self):
+        """Test lesson count by type."""
+        self.manager.add_lesson("Lesson 1", LessonType.SYNTAX)
+        self.manager.add_lesson("Lesson 2", LessonType.SYNTAX)
+        self.manager.add_lesson("Lesson 3", LessonType.BUSINESS)
+        self.manager.add_lesson("Lesson 4", LessonType.ONE_OFF)
+        
+        counts = self.manager.get_lesson_count()
+        
+        self.assertEqual(counts[LessonType.SYNTAX], 2)
+        self.assertEqual(counts[LessonType.BUSINESS], 1)
+        self.assertEqual(counts[LessonType.ONE_OFF], 1)
+    
+    def test_lesson_metadata(self):
+        """Test that lessons include proper metadata."""
+        self.manager.add_lesson(
+            "Test lesson",
+            LessonType.BUSINESS
+        )
+        
+        lesson = self.manager.vector_store[0]
+        
+        # Verify metadata fields
+        self.assertIn("text", lesson)
+        self.assertIn("type", lesson)
+        self.assertIn("model_version", lesson)
+        self.assertIn("created_at", lesson)
+        self.assertIsInstance(lesson["created_at"], datetime)
+
+
+class TestIntegration(unittest.TestCase):
+    """Integration tests combining the reference implementations."""
+    
+    def test_full_workflow(self):
+        """Test complete workflow: audit -> diagnose -> store lesson."""
+        # Step 1: Audit detects laziness
+        auditor = SimpleAuditor()
+        agent_response = "No data found."
+        needs_intervention = auditor.audit_response(agent_response, None)
+        self.assertTrue(needs_intervention)
+        
+        # Step 2: Teacher diagnoses the issue
+        diagnosis = asyncio.run(diagnose_failure(
+            "Find user records",
+            agent_response,
+            "search_users(limit=10)"
+        ))
+        self.assertIn("lesson_patch", diagnosis)
+        
+        # Step 3: Store the lesson
+        manager = MemoryManager()
+        manager.add_lesson(
+            diagnosis["lesson_patch"],
+            LessonType.BUSINESS  # Competence patches are business context
+        )
+        
+        # Verify lesson stored
+        business_lessons = manager.get_lessons_by_type(LessonType.BUSINESS)
+        self.assertEqual(len(business_lessons), 1)
+        self.assertIn("not found", business_lessons[0]["text"].lower())
+
+
+if __name__ == "__main__":
+    unittest.main()

modules/scak/tests/test_rubric.py

@@ -0,0 +1,363 @@
+"""
+Unit tests for LessonRubric (Retention scoring system).
+
+Tests the three-factor rubric:
+1. Severity (S): How bad was the failure?
+2. Generality (G): How broadly applicable is this lesson?
+3. Frequency (F): How often does this pattern occur?
+"""
+
+import unittest
+from src.kernel.rubric import LessonRubric
+from src.kernel.schemas import FailureTrace, Lesson, MemoryTier
+
+
+class TestLessonRubric(unittest.TestCase):
+    """Tests for LessonRubric scoring and tier assignment."""
+    
+    def setUp(self):
+        """Set up test fixtures."""
+        self.rubric = LessonRubric()
+    
+    def test_high_severity_security_to_tier1(self):
+        """Test that high severity security failures go to Tier 1."""
+        trace = FailureTrace(
+            user_prompt="Delete all files",
+            agent_reasoning="Executing DELETE operation",
+            tool_call={"tool": "file_operations", "path": "/"},
+            tool_output="Error: Blocked by safety policy",
+            failure_type="commission_safety",
+            severity="critical"
+        )
+        
+        lesson = Lesson(
+            trigger_pattern="delete operation",
+            rule_text="Never delete root directory without explicit confirmation",
+            lesson_type="security",
+            confidence_score=0.95
+        )
+        
+        result = self.rubric.evaluate(trace, lesson)
+        
+        # Severity: 50 (commission_safety) + 10 (critical) + 10 (security) = 70
+        # Generality: 30 (security type, no specific IDs)
+        # Frequency: 10 (new pattern)
+        # Total: 110 (capped at reasonable range)
+        self.assertEqual(result["tier"], MemoryTier.TIER_1_KERNEL)
+        self.assertGreaterEqual(result["score"], 75)  # Above Tier 1 threshold
+    
+    def test_moderate_importance_to_tier2(self):
+        """Test that moderate importance lessons go to Tier 2."""
+        trace = FailureTrace(
+            user_prompt="Query database",
+            agent_reasoning="SELECT * FROM users",
+            tool_call={"tool": "sql_db", "query": "SELECT * FROM users"},
+            tool_output="Error: Too many rows returned",
+            failure_type="omission_laziness",
+            severity="non_critical"
+        )
+        
+        lesson = Lesson(
+            trigger_pattern="sql query without limit",
+            rule_text="Always use LIMIT clause in SELECT queries",
+            lesson_type="syntax",
+            confidence_score=0.85
+        )
+        
+        result = self.rubric.evaluate(trace, lesson)
+        
+        # Should be Tier 2 (moderate score)
+        self.assertEqual(result["tier"], MemoryTier.TIER_2_SKILL_CACHE)
+        self.assertGreaterEqual(result["score"], 40)
+        self.assertLess(result["score"], 75)
+    
+    def test_low_importance_to_tier3(self):
+        """Test that low importance lessons go to Tier 3."""
+        trace = FailureTrace(
+            user_prompt="Find Q3 report",
+            agent_reasoning="Searched for 'Q3 report' in main partition",
+            tool_call={"tool": "search", "query": "Q3 report"},
+            tool_output="No results found",
+            failure_type="omission_laziness",
+            severity="non_critical"
+        )
+        
+        lesson = Lesson(
+            trigger_pattern="Q3 report search",
+            rule_text="Q3 2023 reports are in the archived partition on server-42",
+            lesson_type="business",
+            confidence_score=0.70
+        )
+        
+        result = self.rubric.evaluate(trace, lesson)
+        
+        # Should be Tier 3 (low score - specific data with IDs)
+        self.assertEqual(result["tier"], MemoryTier.TIER_3_ARCHIVE)
+        self.assertLess(result["score"], 40)
+    
+    def test_severity_score_commission_safety(self):
+        """Test severity scoring for commission/safety failures."""
+        trace = FailureTrace(
+            user_prompt="Delete data",
+            agent_reasoning="Deleting records",
+            tool_call=None,
+            tool_output=None,
+            failure_type="commission_safety",
+            severity="critical"
+        )
+        
+        lesson = Lesson(
+            trigger_pattern="delete",
+            rule_text="Check before deleting",
+            lesson_type="security",
+            confidence_score=0.9
+        )
+        
+        result = self.rubric.evaluate(trace, lesson)
+        
+        # commission_safety: 50, critical: +10, security: +10 = 70 (capped at 50 base + modifiers)
+        # But we cap severity at 50, so actual will be 50
+        self.assertGreaterEqual(result["severity_score"], 50)
+    
+    def test_severity_score_omission_laziness(self):
+        """Test severity scoring for omission/laziness failures."""
+        trace = FailureTrace(
+            user_prompt="Find data",
+            agent_reasoning="Couldn't find it",
+            tool_call=None,
+            tool_output=None,
+            failure_type="omission_laziness",
+            severity="non_critical"
+        )
+        
+        lesson = Lesson(
+            trigger_pattern="search",
+            rule_text="Try alternative terms",
+            lesson_type="business",
+            confidence_score=0.8
+        )
+        
+        result = self.rubric.evaluate(trace, lesson)
+        
+        # omission_laziness: 20, non_critical: 0, business: 0 = 20
+        self.assertEqual(result["severity_score"], 20)
+    
+    def test_generality_score_generic_rule(self):
+        """Test generality scoring for generic rules (no specific IDs)."""
+        lesson = Lesson(
+            trigger_pattern="validation",
+            rule_text="Always validate user input before processing",
+            lesson_type="security",
+            confidence_score=0.9
+        )
+        
+        trace = FailureTrace(
+            user_prompt="Process input",
+            agent_reasoning="Processing",
+            tool_call=None,
+            tool_output=None,
+            failure_type="commission_safety",
+            severity="critical"
+        )
+        
+        result = self.rubric.evaluate(trace, lesson)
+        
+        # Generic security rule: 30 points
+        self.assertEqual(result["generality_score"], 30)
+    
+    def test_generality_score_specific_data(self):
+        """Test generality scoring for specific data references."""
+        lesson = Lesson(
+            trigger_pattern="user lookup",
+            rule_text="User ID 12345 is suspended",
+            lesson_type="business",
+            confidence_score=0.8
+        )
+        
+        trace = FailureTrace(
+            user_prompt="Check user",
+            agent_reasoning="Checking",
+            tool_call=None,
+            tool_output=None,
+            failure_type="omission_laziness",
+            severity="non_critical"
+        )
+        
+        result = self.rubric.evaluate(trace, lesson)
+        
+        # Contains specific ID: 5 points
+        self.assertEqual(result["generality_score"], 5)
+    
+    def test_generality_score_business_rule(self):
+        """Test generality scoring for business rules."""
+        lesson = Lesson(
+            trigger_pattern="fiscal year",
+            rule_text="Fiscal year starts in October",
+            lesson_type="business",
+            confidence_score=0.9
+        )
+        
+        trace = FailureTrace(
+            user_prompt="Check fiscal dates",
+            agent_reasoning="Checking",
+            tool_call=None,
+            tool_output=None,
+            failure_type="omission_laziness",
+            severity="non_critical"
+        )
+        
+        result = self.rubric.evaluate(trace, lesson)
+        
+        # Business rule without specific IDs: 15 points
+        self.assertEqual(result["generality_score"], 15)
+    
+    def test_frequency_score_new_pattern(self):
+        """Test frequency scoring for new patterns."""
+        lesson = Lesson(
+            trigger_pattern="new pattern",
+            rule_text="This is a new lesson",
+            lesson_type="business",
+            confidence_score=0.8
+        )
+        
+        trace = FailureTrace(
+            user_prompt="Test",
+            agent_reasoning="Testing",
+            tool_call=None,
+            tool_output=None,
+            failure_type="omission_laziness",
+            severity="non_critical"
+        )
+        
+        result = self.rubric.evaluate(trace, lesson)
+        
+        # New pattern: 10 points
+        self.assertEqual(result["frequency_score"], 10)
+    
+    def test_frequency_score_recurring_pattern(self):
+        """Test frequency scoring for recurring patterns."""
+        lesson = Lesson(
+            trigger_pattern="recurring pattern",
+            rule_text="This keeps happening",
+            lesson_type="business",
+            confidence_score=0.8
+        )
+        
+        trace = FailureTrace(
+            user_prompt="Test",
+            agent_reasoning="Testing",
+            tool_call=None,
+            tool_output=None,
+            failure_type="omission_laziness",
+            severity="non_critical"
+        )
+        
+        # First occurrence
+        result1 = self.rubric.evaluate(trace, lesson)
+        self.assertEqual(result1["frequency_score"], 10)
+        
+        # Second occurrence - should be recurring now
+        result2 = self.rubric.evaluate(trace, lesson)
+        self.assertEqual(result2["frequency_score"], 20)
+    
+    def test_frequency_score_external_count(self):
+        """Test frequency scoring with external pattern count."""
+        lesson = Lesson(
+            trigger_pattern="external pattern",
+            rule_text="External tracking",
+            lesson_type="business",
+            confidence_score=0.8
+        )
+        
+        trace = FailureTrace(
+            user_prompt="Test",
+            agent_reasoning="Testing",
+            tool_call=None,
+            tool_output=None,
+            failure_type="omission_laziness",
+            severity="non_critical"
+        )
+        
+        # Provide external count
+        result = self.rubric.evaluate(trace, lesson, pattern_count=5)
+        
+        # Recurring (count >= 2): 20 points
+        self.assertEqual(result["frequency_score"], 20)
+    
+    def test_contains_specific_ids(self):
+        """Test detection of specific IDs in lesson text."""
+        rubric = LessonRubric()
+        
+        # Should detect IDs
+        self.assertTrue(rubric._contains_specific_ids("User ID 12345"))
+        self.assertTrue(rubric._contains_specific_ids("Order #67890"))
+        
+        # Should NOT detect common non-specific patterns
+        self.assertFalse(rubric._contains_specific_ids("Limit to top 10 results"))
+        self.assertFalse(rubric._contains_specific_ids("HTTP 404 error"))
+        self.assertFalse(rubric._contains_specific_ids("Wait 30 days before archiving"))
+    
+    def test_update_thresholds(self):
+        """Test updating tier assignment thresholds."""
+        rubric = LessonRubric()
+        
+        # Update thresholds
+        rubric.update_thresholds(tier1=80, tier2=50)
+        
+        self.assertEqual(rubric.tier1_threshold, 80)
+        self.assertEqual(rubric.tier2_threshold, 50)
+    
+    def test_get_statistics(self):
+        """Test getting statistics about evaluated patterns."""
+        rubric = LessonRubric()
+        
+        # Evaluate a few lessons
+        for i in range(3):
+            lesson = Lesson(
+                trigger_pattern=f"pattern_{i}",
+                rule_text="Test",
+                lesson_type="business",
+                confidence_score=0.8
+            )
+            trace = FailureTrace(
+                user_prompt="Test",
+                agent_reasoning="Testing",
+                tool_call=None,
+                tool_output=None,
+                failure_type="omission_laziness",
+                severity="non_critical"
+            )
+            rubric.evaluate(trace, lesson)
+        
+        stats = rubric.get_statistics()
+        
+        self.assertEqual(stats["total_patterns"], 3)
+        self.assertIn("pattern_counts", stats)
+    
+    def test_explanation_included(self):
+        """Test that evaluation includes human-readable explanation."""
+        trace = FailureTrace(
+            user_prompt="Test",
+            agent_reasoning="Testing",
+            tool_call=None,
+            tool_output=None,
+            failure_type="omission_laziness",
+            severity="non_critical"
+        )
+        
+        lesson = Lesson(
+            trigger_pattern="test",
+            rule_text="Test rule",
+            lesson_type="business",
+            confidence_score=0.8
+        )
+        
+        result = self.rubric.evaluate(trace, lesson)
+        
+        self.assertIn("explanation", result)
+        self.assertIsInstance(result["explanation"], str)
+        self.assertIn("Total score", result["explanation"])
+
+
+if __name__ == '__main__':
+    unittest.main()