agent-os-kernel 1.1.0__py3-none-any.whl → 1.2.0__py3-none-any.whl

modules/mute-agent/experiments/v2_scenarios/scenario_3_false_positive.py

@@ -0,0 +1,303 @@
+"""
+Scenario Suite 3: The False Positive Trap
+
+Tests the boundary between "Safety" and "Friction".
+Scenario C: The "Synonym" Stress Test
+"""
+
+from typing import Dict, Any, List
+from dataclasses import dataclass
+import sys
+import os
+
+# Add parent directory to path to allow imports when running this script directly
+# This is necessary because experiments can be run from various working directories
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), '../..'))
+
+from mute_agent import (
+    ReasoningAgent,
+    ExecutionAgent,
+    HandshakeProtocol,
+    MultidimensionalKnowledgeGraph,
+    SuperSystemRouter,
+)
+from mute_agent.knowledge_graph.graph_elements import Node, NodeType, Edge, EdgeType
+from mute_agent.knowledge_graph.subgraph import Dimension
+
+
+@dataclass
+class SynonymTestResult:
+    """Result of a synonym normalization test."""
+    scenario_name: str
+    user_input: str
+    canonical_value: str
+    was_normalized: bool
+    was_rejected: bool
+    is_false_positive: bool
+    error_message: str
+
+
+# Test cases with colloquial phrasings
+SYNONYM_TEST_CASES = [
+    # Region synonyms
+    {"input": "Virginia", "field": "region", "canonical": "us-east-1", "valid_intent": True},
+    {"input": "N. Virginia", "field": "region", "canonical": "us-east-1", "valid_intent": True},
+    {"input": "Northern Virginia", "field": "region", "canonical": "us-east-1", "valid_intent": True},
+    {"input": "Oregon", "field": "region", "canonical": "us-west-2", "valid_intent": True},
+    {"input": "California", "field": "region", "canonical": "us-west-1", "valid_intent": True},
+    {"input": "Ohio", "field": "region", "canonical": "us-east-2", "valid_intent": True},
+    {"input": "us-east", "field": "region", "canonical": "us-east-1", "valid_intent": True},
+    {"input": "East", "field": "region", "canonical": "us-east-1", "valid_intent": True},
+    {"input": "West", "field": "region", "canonical": "us-west-2", "valid_intent": True},
+    
+    # Environment synonyms
+    {"input": "production", "field": "env", "canonical": "prod", "valid_intent": True},
+    {"input": "the prod env", "field": "env", "canonical": "prod", "valid_intent": True},
+    {"input": "production environment", "field": "env", "canonical": "prod", "valid_intent": True},
+    {"input": "live", "field": "env", "canonical": "prod", "valid_intent": True},
+    {"input": "development", "field": "env", "canonical": "dev", "valid_intent": True},
+    {"input": "the dev env", "field": "env", "canonical": "dev", "valid_intent": True},
+    {"input": "dev env", "field": "env", "canonical": "dev", "valid_intent": True},
+    {"input": "staging", "field": "env", "canonical": "stage", "valid_intent": True},
+    
+    # Invalid inputs (should be rejected)
+    {"input": "Mars", "field": "region", "canonical": None, "valid_intent": False},
+    {"input": "xyz123", "field": "region", "canonical": None, "valid_intent": False},
+    {"input": "production123", "field": "env", "canonical": None, "valid_intent": False},
+]
+
+
+class SynonymScenario:
+    """
+    Scenario C: The "Synonym" Stress Test
+    
+    Graph Requirement: Region must be one of ['us-east-1', 'us-west-2', 'us-west-1', 'us-east-2']
+    User Command: "Spin this up in Virginia"
+    
+    Test: Does the Mute Agent reject "Virginia" or normalize it to "us-east-1"?
+    """
+    
+    def __init__(self):
+        self.kg = MultidimensionalKnowledgeGraph()
+        self.router = SuperSystemRouter(self.kg)
+        self.protocol = HandshakeProtocol()
+        self.reasoning_agent = ReasoningAgent(self.kg, self.router, self.protocol)
+        self.execution_agent = ExecutionAgent(self.protocol)
+        
+        # Setup the resource graph
+        self._setup_resource_graph()
+    
+    def _setup_resource_graph(self):
+        """Setup the resource dimension with region and environment constraints."""
+        # Create resource dimension
+        resource_dim = Dimension(
+            name="resources",
+            description="Resource allocation constraints",
+            priority=8
+        )
+        self.kg.add_dimension(resource_dim)
+        
+        # Create nodes for different regions
+        deploy_action = Node(
+            id="deploy_resource",
+            node_type=NodeType.ACTION,
+            attributes={"operation": "deploy"}
+        )
+        
+        region_constraint = Node(
+            id="region_constraint",
+            node_type=NodeType.CONSTRAINT,
+            attributes={
+                "required": True,
+                "type": "region",
+                "allowed_values": ["us-east-1", "us-west-2", "us-west-1", "us-east-2"]
+            }
+        )
+        
+        env_constraint = Node(
+            id="environment_constraint",
+            node_type=NodeType.CONSTRAINT,
+            attributes={
+                "required": True,
+                "type": "environment",
+                "allowed_values": ["prod", "dev", "stage", "test"]
+            }
+        )
+        
+        # Add nodes to graph
+        self.kg.add_node_to_dimension("resources", deploy_action)
+        self.kg.add_node_to_dimension("resources", region_constraint)
+        self.kg.add_node_to_dimension("resources", env_constraint)
+        
+        # Create constraints
+        edge1 = Edge(
+            source_id="deploy_resource",
+            target_id="region_constraint",
+            edge_type=EdgeType.REQUIRES,
+            weight=1.0
+        )
+        
+        edge2 = Edge(
+            source_id="deploy_resource",
+            target_id="environment_constraint",
+            edge_type=EdgeType.REQUIRES,
+            weight=1.0
+        )
+        
+        self.kg.add_edge_to_dimension("resources", edge1)
+        self.kg.add_edge_to_dimension("resources", edge2)
+    
+    def test_synonym_normalization(self, user_input: str, field: str, canonical: str, valid_intent: bool) -> SynonymTestResult:
+        """
+        Test if the router normalizes a colloquial input to canonical value.
+        """
+        # Create context with user's colloquial input
+        context = {
+            "user": "developer",
+            "authenticated": True,
+            field: user_input,
+            "region_constraint_satisfied": True,
+            "environment_constraint_satisfied": True
+        }
+        
+        # Route the context (normalization happens here)
+        routing_result = self.router.route(context)
+        
+        # Check if normalization occurred
+        normalized_context = routing_result.routing_metadata.get("normalized_context", context)
+        was_normalized = normalized_context.get(field) != user_input
+        
+        # Check if the canonical value matches expected
+        normalized_value = normalized_context.get(field)
+        correct_normalization = normalized_value == canonical if canonical else False
+        
+        # Attempt to execute action
+        session = self.reasoning_agent.propose_action(
+            action_id="deploy_resource",
+            parameters={"resource_type": "server"},
+            context=context,
+            justification=f"User requested deployment in {user_input}"
+        )
+        
+        validation_result = session.validation_result
+        was_rejected = not validation_result.is_valid if validation_result else True
+        
+        # False positive = valid intent but rejected
+        is_false_positive = valid_intent and was_rejected and not was_normalized
+        
+        error_message = "; ".join(validation_result.errors) if validation_result else ""
+        
+        return SynonymTestResult(
+            scenario_name="Synonym Normalization",
+            user_input=user_input,
+            canonical_value=canonical if canonical else "N/A",
+            was_normalized=was_normalized,
+            was_rejected=was_rejected,
+            is_false_positive=is_false_positive,
+            error_message=error_message
+        )
+
+
+def run_synonym_experiment():
+    """Run the synonym stress test experiment."""
+    print("=" * 80)
+    print("SCENARIO SUITE 3: THE FALSE POSITIVE TRAP")
+    print("=" * 80)
+    print()
+    
+    scenario = SynonymScenario()
+    
+    print("Test: Synonym Stress Test (Colloquial Phrasings)")
+    print("-" * 80)
+    print()
+    
+    results = []
+    false_positives = 0
+    true_negatives = 0
+    true_positives = 0
+    false_negatives = 0
+    
+    for i, test_case in enumerate(SYNONYM_TEST_CASES, 1):
+        result = scenario.test_synonym_normalization(
+            test_case["input"],
+            test_case["field"],
+            test_case["canonical"],
+            test_case["valid_intent"]
+        )
+        results.append(result)
+        
+        # Categorize result
+        if result.is_false_positive:
+            false_positives += 1
+            status = "✗ FALSE POSITIVE"
+        elif test_case["valid_intent"] and not result.was_rejected:
+            true_positives += 1
+            status = "✓ CORRECT (Accepted)"
+        elif not test_case["valid_intent"] and result.was_rejected:
+            true_negatives += 1
+            status = "✓ CORRECT (Rejected)"
+        elif not test_case["valid_intent"] and not result.was_rejected:
+            false_negatives += 1
+            status = "✗ FALSE NEGATIVE"
+        else:
+            status = "? UNCLEAR"
+        
+        print(f"Test {i}: {test_case['field']} = '{result.user_input}'")
+        print(f"  Expected:    {result.canonical_value}")
+        print(f"  Normalized:  {'✓ YES' if result.was_normalized else '✗ NO'}")
+        print(f"  Rejected:    {'YES' if result.was_rejected else 'NO'}")
+        print(f"  Status:      {status}")
+        if result.error_message:
+            print(f"  Error:       {result.error_message[:80]}...")
+        print()
+    
+    # Calculate metrics
+    total_valid_intent = sum(1 for tc in SYNONYM_TEST_CASES if tc["valid_intent"])
+    rejection_rate_on_valid = (false_positives / total_valid_intent * 100) if total_valid_intent > 0 else 0
+    
+    # Summary
+    print("=" * 80)
+    print("EXPERIMENT 3.1: THE FRUSTRATION SCORE")
+    print("=" * 80)
+    print()
+    print("Hypothesis: System should normalize colloquial terms INTO graph values, not reject them.")
+    print("           If it rejects valid synonyms, the 'Mute' approach is too brittle.")
+    print()
+    print(f"Total Test Cases:              {len(SYNONYM_TEST_CASES)}")
+    print(f"Valid Intent Cases:            {total_valid_intent}")
+    print(f"True Positives (Accepted):     {true_positives}")
+    print(f"False Positives (Rejected):    {false_positives}")
+    print(f"True Negatives (Rejected):     {true_negatives}")
+    print(f"False Negatives (Accepted):    {false_negatives}")
+    print()
+    print(f"Rejection Rate on Valid Intent: {rejection_rate_on_valid:.1f}%")
+    print()
+    print(f"Status:                        {'✓ LOW FRICTION' if rejection_rate_on_valid < 20 else '✗ HIGH FRICTION'}")
+    print()
+    
+    # Breakdown by field
+    print("Breakdown by Field:")
+    print("-" * 80)
+    field_stats = {}
+    for tc, result in zip(SYNONYM_TEST_CASES, results):
+        field = tc["field"]
+        if field not in field_stats:
+            field_stats[field] = {"total": 0, "normalized": 0, "false_positive": 0}
+        field_stats[field]["total"] += 1
+        if result.was_normalized:
+            field_stats[field]["normalized"] += 1
+        if result.is_false_positive:
+            field_stats[field]["false_positive"] += 1
+    
+    for field, stats in field_stats.items():
+        norm_rate = (stats["normalized"] / stats["total"] * 100) if stats["total"] > 0 else 0
+        fp_rate = (stats["false_positive"] / stats["total"] * 100) if stats["total"] > 0 else 0
+        print(f"{field:15s}  Normalized: {norm_rate:5.1f}%  False Positives: {fp_rate:5.1f}%")
+    
+    print()
+    
+    return results
+
+
+if __name__ == "__main__":
+    results = run_synonym_experiment()

modules/mute-agent/experiments/v2_scenarios/scenario_4_performance.py

@@ -0,0 +1,319 @@
+"""
+Scenario Suite 4: Performance Experiments
+
+Tests token economics and latency at scale.
+Experiment 4.1: Token Economics Benchmark
+Experiment 4.2: Latency at Scale (Graph Size)
+"""
+
+from typing import Dict, Any, List
+from dataclasses import dataclass
+import sys
+import os
+import time
+
+# Add parent directory to path to allow imports when running this script directly
+# This is necessary because experiments can be run from various working directories
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), '../..'))
+
+from mute_agent import (
+    ReasoningAgent,
+    ExecutionAgent,
+    HandshakeProtocol,
+    MultidimensionalKnowledgeGraph,
+    SuperSystemRouter,
+)
+from mute_agent.knowledge_graph.graph_elements import Node, NodeType, Edge, EdgeType
+from mute_agent.knowledge_graph.subgraph import Dimension
+
+
+@dataclass
+class PerformanceTestResult:
+    """Result of a performance test."""
+    scenario_name: str
+    graph_size: int
+    num_dimensions: int
+    request_type: str  # "complete" or "incomplete"
+    latency_ms: float
+    estimated_tokens: int
+    success: bool
+
+
+class PerformanceScenario:
+    """
+    Performance testing scenarios for Mute Agent.
+    
+    Experiment 4.1: Token Economics
+    - Hypothesis: Mute Agent reduces API costs by 90% for failure cases
+    - Measure: Tokens spent on incomplete commands
+    
+    Experiment 4.2: Latency at Scale
+    - Hypothesis: Graph traversal is O(1) or O(log N) relative to prompt size
+    - Measure: Time to First Token with 10 vs 10,000 nodes
+    """
+    
+    def __init__(self):
+        pass
+    
+    def create_small_graph(self) -> tuple:
+        """Create a small graph with 10 nodes."""
+        kg = MultidimensionalKnowledgeGraph()
+        
+        # Create 2 dimensions
+        for dim_num in range(2):
+            dim = Dimension(
+                name=f"dimension_{dim_num}",
+                description=f"Test dimension {dim_num}",
+                priority=10 - dim_num
+            )
+            kg.add_dimension(dim)
+            
+            # Add 5 nodes per dimension
+            for node_num in range(5):
+                node = Node(
+                    id=f"node_{dim_num}_{node_num}",
+                    node_type=NodeType.ACTION if node_num < 3 else NodeType.CONSTRAINT,
+                    attributes={"test": True, "dim": dim_num}
+                )
+                kg.add_node_to_dimension(f"dimension_{dim_num}", node)
+        
+        router = SuperSystemRouter(kg)
+        protocol = HandshakeProtocol()
+        reasoning_agent = ReasoningAgent(kg, router, protocol)
+        
+        return kg, router, reasoning_agent
+    
+    def create_large_graph(self) -> tuple:
+        """Create a large graph with 10,000 nodes."""
+        kg = MultidimensionalKnowledgeGraph()
+        
+        # Create 20 dimensions
+        for dim_num in range(20):
+            dim = Dimension(
+                name=f"dimension_{dim_num}",
+                description=f"Test dimension {dim_num}",
+                priority=20 - dim_num
+            )
+            kg.add_dimension(dim)
+            
+            # Add 500 nodes per dimension = 10,000 total
+            for node_num in range(500):
+                node = Node(
+                    id=f"node_{dim_num}_{node_num}",
+                    node_type=NodeType.ACTION if node_num < 250 else NodeType.CONSTRAINT,
+                    attributes={"test": True, "dim": dim_num, "node": node_num}
+                )
+                kg.add_node_to_dimension(f"dimension_{dim_num}", node)
+                
+                # Add some edges for realism
+                if node_num > 0 and node_num % 10 == 0:
+                    edge = Edge(
+                        source_id=f"node_{dim_num}_{node_num}",
+                        target_id=f"node_{dim_num}_{node_num - 1}",
+                        edge_type=EdgeType.REQUIRES,
+                        weight=1.0
+                    )
+                    kg.add_edge_to_dimension(f"dimension_{dim_num}", edge)
+        
+        router = SuperSystemRouter(kg)
+        protocol = HandshakeProtocol()
+        reasoning_agent = ReasoningAgent(kg, router, protocol)
+        
+        return kg, router, reasoning_agent
+    
+    def measure_latency(self, reasoning_agent: ReasoningAgent, context: Dict[str, Any], 
+                       action_id: str, graph_size: int, num_dimensions: int) -> PerformanceTestResult:
+        """Measure latency for a single request."""
+        start_time = time.time()
+        
+        # Perform routing and validation
+        session = reasoning_agent.propose_action(
+            action_id=action_id,
+            parameters={"test": True},
+            context=context,
+            justification="Performance test"
+        )
+        
+        end_time = time.time()
+        latency_ms = (end_time - start_time) * 1000
+        
+        # Estimate tokens used (simplified)
+        # Approximation: ~4 characters per token for English text
+        # Mute Agent: context + small validation logic
+        CHARS_PER_TOKEN = 4
+        estimated_tokens = len(str(context)) // CHARS_PER_TOKEN + 50  # ~50 tokens for validation
+        
+        validation_result = session.validation_result
+        success = validation_result.is_valid if validation_result else False
+        request_type = "complete" if success else "incomplete"
+        
+        return PerformanceTestResult(
+            scenario_name="Latency Test",
+            graph_size=graph_size,
+            num_dimensions=num_dimensions,
+            request_type=request_type,
+            latency_ms=latency_ms,
+            estimated_tokens=estimated_tokens,
+            success=success
+        )
+    
+    def test_token_economics(self) -> List[PerformanceTestResult]:
+        """
+        Experiment 4.1: Token Economics Benchmark
+        Run incomplete commands and measure token usage.
+        """
+        results = []
+        
+        # Small graph with incomplete requests
+        kg, router, reasoning_agent = self.create_small_graph()
+        
+        # Test 10 incomplete requests
+        for i in range(10):
+            context = {
+                "user": "test_user",
+                "test_id": i
+                # Missing required fields - intentionally incomplete
+            }
+            
+            result = self.measure_latency(
+                reasoning_agent,
+                context,
+                f"node_0_{i % 3}",
+                graph_size=10,
+                num_dimensions=2
+            )
+            results.append(result)
+        
+        return results
+    
+    def test_latency_at_scale(self) -> List[PerformanceTestResult]:
+        """
+        Experiment 4.2: Latency at Scale
+        Compare latency with small (10 nodes) vs large (10,000 nodes) graphs.
+        """
+        results = []
+        
+        # Test with small graph
+        print("Creating small graph (10 nodes)...")
+        kg_small, router_small, agent_small = self.create_small_graph()
+        
+        context_small = {
+            "user": "test_user",
+            "authenticated": True
+        }
+        
+        # Warm-up run
+        _ = self.measure_latency(agent_small, context_small, "node_0_0", 10, 2)
+        
+        # Measure 5 times and average
+        for i in range(5):
+            result = self.measure_latency(agent_small, context_small, "node_0_0", 10, 2)
+            results.append(result)
+        
+        # Test with large graph
+        print("Creating large graph (10,000 nodes)...")
+        kg_large, router_large, agent_large = self.create_large_graph()
+        
+        context_large = {
+            "user": "test_user",
+            "authenticated": True
+        }
+        
+        # Warm-up run
+        _ = self.measure_latency(agent_large, context_large, "node_0_0", 10000, 20)
+        
+        # Measure 5 times and average
+        for i in range(5):
+            result = self.measure_latency(agent_large, context_large, "node_0_0", 10000, 20)
+            results.append(result)
+        
+        return results
+
+
+def run_performance_experiments():
+    """Run all performance experiments."""
+    print("=" * 80)
+    print("SCENARIO SUITE 4: PERFORMANCE EXPERIMENTS")
+    print("=" * 80)
+    print()
+    
+    scenario = PerformanceScenario()
+    
+    # Experiment 4.1: Token Economics
+    print("EXPERIMENT 4.1: TOKEN ECONOMICS BENCHMARK")
+    print("-" * 80)
+    print()
+    print("Hypothesis: Mute Agent reduces API costs by 90% for failure cases.")
+    print("           Failing fast is nearly free.")
+    print()
+    
+    token_results = scenario.test_token_economics()
+    
+    total_tokens = sum(r.estimated_tokens for r in token_results)
+    avg_tokens = total_tokens / len(token_results) if token_results else 0
+    avg_latency = sum(r.latency_ms for r in token_results) / len(token_results) if token_results else 0
+    
+    # Baseline comparison (simulated ReAct agent)
+    # ReAct agent uses ~1250 tokens per incomplete request (from existing experiments)
+    baseline_tokens = 1250 * len(token_results)
+    token_reduction = ((baseline_tokens - total_tokens) / baseline_tokens * 100) if baseline_tokens > 0 else 0
+    
+    print(f"Number of Requests:        {len(token_results)}")
+    print(f"Total Tokens (Mute):       {total_tokens}")
+    print(f"Avg Tokens per Request:    {avg_tokens:.0f}")
+    print(f"Avg Latency:               {avg_latency:.2f}ms")
+    print()
+    print(f"Baseline (ReAct) Total:    {baseline_tokens}")
+    print(f"Token Reduction:           {token_reduction:.1f}%")
+    print()
+    print(f"Status:                    {'✓ HYPOTHESIS VALIDATED' if token_reduction >= 70 else '✗ HYPOTHESIS REJECTED'}")
+    print()
+    
+    # Experiment 4.2: Latency at Scale
+    print("=" * 80)
+    print("EXPERIMENT 4.2: LATENCY AT SCALE (GRAPH SIZE)")
+    print("=" * 80)
+    print()
+    print("Hypothesis: Graph traversal is O(1) or O(log N) relative to graph size.")
+    print("           Mute Agent speed remains constant as graph grows.")
+    print()
+    
+    scale_results = scenario.test_latency_at_scale()
+    
+    # Separate results by graph size
+    small_results = [r for r in scale_results if r.graph_size == 10]
+    large_results = [r for r in scale_results if r.graph_size == 10000]
+    
+    avg_latency_small = sum(r.latency_ms for r in small_results) / len(small_results) if small_results else 0
+    avg_latency_large = sum(r.latency_ms for r in large_results) / len(large_results) if large_results else 0
+    
+    # Calculate scaling factor
+    scaling_factor = avg_latency_large / avg_latency_small if avg_latency_small > 0 else 0
+    graph_size_multiplier = 1000  # 10,000 / 10
+    
+    # O(1) would have scaling_factor ~1
+    # O(log N) would have scaling_factor ~log(1000) ~= 3
+    # O(N) would have scaling_factor ~1000
+    
+    complexity_class = "O(1)" if scaling_factor < 2 else ("O(log N)" if scaling_factor < 10 else "O(N) or worse")
+    
+    print(f"Small Graph (10 nodes):")
+    print(f"  Runs:             {len(small_results)}")
+    print(f"  Avg Latency:      {avg_latency_small:.2f}ms")
+    print()
+    print(f"Large Graph (10,000 nodes):")
+    print(f"  Runs:             {len(large_results)}")
+    print(f"  Avg Latency:      {avg_latency_large:.2f}ms")
+    print()
+    print(f"Graph Size Increase:  {graph_size_multiplier}x")
+    print(f"Latency Increase:     {scaling_factor:.2f}x")
+    print(f"Complexity Class:     {complexity_class}")
+    print()
+    print(f"Status:               {'✓ HYPOTHESIS VALIDATED' if scaling_factor < 10 else '✗ HYPOTHESIS REJECTED'}")
+    print()
+    
+    return token_results + scale_results
+
+
+if __name__ == "__main__":
+    results = run_performance_experiments()