crca 1.4.0__py3-none-any.whl → 1.5.0__py3-none-any.whl

tests/test_crca_llm_coauthor.py

@@ -0,0 +1,12 @@
+from crca_llm import LLMCoauthor
+from crca_core.models.spec import DraftSpec, LockedSpec
+
+
+def test_llm_coauthor_returns_draft_specs_only() -> None:
+    coauthor = LLMCoauthor()
+    bundle = coauthor.draft_specs(user_text="Study effect of X on Y", observed_columns=["X", "Y", "Z"])
+    assert bundle.drafts
+    assert all(isinstance(d, DraftSpec) for d in bundle.drafts)
+    # Ensure it never returns a locked spec
+    assert not any(isinstance(d, LockedSpec) for d in bundle.drafts)
+

tests/test_crca_llm_orchestrator.py

@@ -0,0 +1,80 @@
+import os
+
+import pandas as pd
+
+from crca_core.core.lifecycle import lock_spec
+from crca_core.models.spec import CausalGraphSpec, DraftSpec, EdgeSpec, NodeSpec, RoleSpec
+from crca_core.models.refusal import RefusalResult
+from crca_llm.orchestrator import LLMOrchestrator
+
+
+class FakeClient:
+    def __init__(self, content: str):
+        self._content = content
+
+    def chat_completion(self, **kwargs) -> str:
+        return self._content
+
+
+def test_orchestrator_refuses_without_api_key() -> None:
+    old = os.environ.pop("OPENAI_API_KEY", None)
+    try:
+        orch = LLMOrchestrator()
+        res = orch.run(user_text="Test", observed_columns=["X", "Y"])
+        assert res.refusals
+        assert not res.draft_bundle.drafts
+        assert isinstance(res.refusals[0], RefusalResult)
+    finally:
+        if old is not None:
+            os.environ["OPENAI_API_KEY"] = old
+
+
+def test_orchestrator_never_emits_locked_spec() -> None:
+    payload = {
+        "drafts": [
+            {
+                "nodes": ["X", "Y"],
+                "edges": [["X", "Y"]],
+                "treatments": ["X"],
+                "outcomes": ["Y"],
+                "columns": ["X", "Y"],
+            }
+        ],
+        "review_checklist": ["Confirm time ordering"],
+    }
+    orch = LLMOrchestrator(client=FakeClient(content=str(payload).replace("'", '"')))
+    res = orch.run(user_text="Test", observed_columns=["X", "Y"])
+    assert res.draft_bundle.drafts
+    assert all(d.status.value == "draft" for d in res.draft_bundle.drafts)
+
+
+def test_orchestrator_refuses_estimate_without_identification() -> None:
+    payload = {
+        "drafts": [
+            {
+                "nodes": ["X", "Y"],
+                "edges": [["X", "Y"]],
+                "treatments": ["X"],
+                "outcomes": ["Y"],
+                "columns": ["X", "Y"],
+            }
+        ],
+        "review_checklist": [],
+    }
+    orch = LLMOrchestrator(client=FakeClient(content=str(payload).replace("'", '"')))
+
+    draft = DraftSpec(
+        graph=CausalGraphSpec(nodes=[NodeSpec(name="X"), NodeSpec(name="Y")], edges=[EdgeSpec(source="X", target="Y")]),
+        roles=RoleSpec(treatments=["X"], outcomes=["Y"]),
+    )
+    locked = lock_spec(draft, approvals=["human"])
+    df = pd.DataFrame({"X": [1, 2, 3], "Y": [2, 3, 4]})
+
+    res = orch.run(
+        user_text="Test",
+        observed_columns=["X", "Y"],
+        locked_spec=locked,
+        data=df,
+        actions=["estimate"],
+    )
+    assert res.refusals

tests/test_hybrid_agent_llm_enhanced.py

@@ -0,0 +1,556 @@
+"""
+Comprehensive tests for LLM-enhanced hybrid agent capabilities.
+
+Tests all new features:
+- Conversation memory and context management
+- Chain-of-thought reasoning
+- Few-shot learning
+- Task decomposition
+- Explanation generation
+- Self-verification
+- Consistency guarantees
+- Causal validation
+"""
+
+import os
+import sys
+import pytest
+from pathlib import Path
+
+# Add parent directory to path
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+try:
+    from architecture.hybrid.hybrid_agent import HybridAgent
+    from architecture.hybrid.conversation_manager import ConversationHistory, ContextTracker
+    from architecture.hybrid.reasoning_tracker import ReasoningTracker
+    from architecture.hybrid.few_shot_learner import ExampleStore, PatternLearner, AdaptiveExtractor
+    from architecture.hybrid.task_decomposer import TaskAnalyzer, SubTaskExecutor, PlanGenerator
+    from architecture.hybrid.explanation_generator import ExplanationBuilder, TransparencyLayer
+    from architecture.hybrid.self_verifier import ConsistencyChecker, ErrorDetector, SelfCorrector
+    from architecture.hybrid.consistency_engine import ConsistencyEngine, DeterministicProcessor, StateSnapshot
+    from schemas.conversation import ConversationContext, MessageRole, GraphSnapshot
+    from schemas.reasoning import ReasoningChain, StepType, InferenceRule, Evidence
+    HYBRID_AGENT_AVAILABLE = True
+except ImportError as e:
+    HYBRID_AGENT_AVAILABLE = False
+    pytest.skip(f"Hybrid agent not available: {e}", allow_module_level=True)
+
+
+class TestConversationManagement:
+    """Test conversation memory and context management."""
+    
+    def test_conversation_history_creation(self):
+        """Test creating conversation history."""
+        history = ConversationHistory()
+        assert history.conversation_id is not None
+        assert len(history.context.messages) == 0
+    
+    def test_add_message(self):
+        """Test adding messages to conversation."""
+        history = ConversationHistory()
+        message = history.add_message(MessageRole.USER, "What affects price?")
+        assert message.role == MessageRole.USER
+        assert message.content == "What affects price?"
+        assert len(history.context.messages) == 1
+    
+    def test_attention_weights(self):
+        """Test attention weight computation."""
+        history = ConversationHistory(decay_lambda=0.1)
+        history.add_message(MessageRole.USER, "Message 1")
+        history.add_message(MessageRole.AGENT, "Response 1")
+        history.add_message(MessageRole.USER, "Message 2")
+        
+        weights = history.context.compute_attention_weights()
+        assert len(weights) == 3
+        # Most recent message should have highest weight
+        assert weights[2] > weights[0]
+    
+    def test_context_retrieval(self):
+        """Test context retrieval."""
+        history = ConversationHistory()
+        history.add_message(MessageRole.USER, "What affects price?")
+        history.add_message(MessageRole.AGENT, "Price depends on demand and supply")
+        history.add_message(MessageRole.USER, "How about demand?")
+        
+        context = history.retrieve_context(k=2)
+        assert len(context) <= 2
+        assert all(isinstance(msg, type(history.context.messages[0])) for msg in context)
+    
+    def test_context_tracker(self):
+        """Test context tracker."""
+        history = ConversationHistory()
+        tracker = ContextTracker(history)
+        
+        history.add_message(MessageRole.USER, "What affects price?")
+        history.add_message(MessageRole.AGENT, "Price depends on demand")
+        
+        relevant = tracker.get_relevant_context("price", k=2)
+        assert len(relevant) <= 2
+    
+    def test_reference_resolution(self):
+        """Test reference resolution."""
+        history = ConversationHistory()
+        tracker = ContextTracker(history)
+        
+        history.add_message(MessageRole.USER, "What affects price?")
+        history.add_message(MessageRole.AGENT, "Price depends on demand")
+        history.add_message(MessageRole.USER, "How about it?")
+        
+        resolved = tracker.resolve_reference("it", history.context.current_turn)
+        # Should resolve to "price" or "demand"
+        assert resolved is not None or resolved is None  # May or may not resolve
+
+
+class TestReasoningTracking:
+    """Test chain-of-thought reasoning tracking."""
+    
+    def test_reasoning_tracker_creation(self):
+        """Test creating reasoning tracker."""
+        tracker = ReasoningTracker()
+        assert tracker.chains == {}
+        assert tracker.current_chain is None
+    
+    def test_create_chain(self):
+        """Test creating reasoning chain."""
+        tracker = ReasoningTracker()
+        chain = tracker.create_chain()
+        assert chain is not None
+        assert chain.chain_id is not None
+        assert tracker.current_chain == chain
+    
+    def test_add_step(self):
+        """Test adding reasoning steps."""
+        tracker = ReasoningTracker()
+        tracker.create_chain()
+        
+        step = tracker.add_step(
+            step_type=StepType.EXTRACTION,
+            operation="extract_variables",
+            input_state={'task': 'test'},
+            output_state={'variables': ['x', 'y']},
+            conclusion="Extracted 2 variables"
+        )
+        
+        assert step is not None
+        assert step.step_id is not None
+        assert len(tracker.current_chain.steps) == 1
+    
+    def test_chain_validation(self):
+        """Test reasoning chain validation."""
+        tracker = ReasoningTracker()
+        chain = tracker.create_chain()
+        
+        # Add valid step
+        tracker.add_step(
+            step_type=StepType.EXTRACTION,
+            operation="extract",
+            input_state={},
+            output_state={},
+            conclusion="test",
+            evidence=[Evidence(source="test", content="evidence")]
+        )
+        
+        is_valid, error = tracker.validate_chain()
+        assert is_valid or error is not None  # May be valid or have error
+
+
+class TestFewShotLearning:
+    """Test few-shot learning capabilities."""
+    
+    def test_example_store(self):
+        """Test example store."""
+        store = ExampleStore()
+        store.add_example("price depends on demand", {"variables": ["price", "demand"], "edges": [("demand", "price")]})
+        assert len(store.examples) == 1
+    
+    def test_find_similar_examples(self):
+        """Test finding similar examples."""
+        store = ExampleStore()
+        store.add_example("price depends on demand", {"variables": ["price", "demand"]})
+        store.add_example("cost affects profit", {"variables": ["cost", "profit"]})
+        
+        similar = store.find_similar_examples("price relates to demand", k=1)
+        assert len(similar) <= 1
+        if similar:
+            assert "price" in similar[0][0].lower() or "demand" in similar[0][0].lower()
+    
+    def test_pattern_learning(self):
+        """Test pattern learning."""
+        store = ExampleStore()
+        learner = PatternLearner(store)
+        
+        examples = [
+            ("price depends on demand", {"variables": ["price", "demand"], "edges": [("demand", "price")]}),
+            ("cost affects profit", {"variables": ["cost", "profit"], "edges": [("cost", "profit")]})
+        ]
+        
+        patterns = learner.learn_from_examples(examples)
+        assert len(patterns) > 0
+    
+    def test_adaptive_extraction(self):
+        """Test adaptive extraction."""
+        store = ExampleStore()
+        learner = PatternLearner(store)
+        extractor = AdaptiveExtractor(learner, store)
+        
+        # Add examples
+        store.add_example("price depends on demand", {"variables": ["price", "demand"], "edges": [("demand", "price")]})
+        learner.learn_from_examples()
+        
+        # Try extraction
+        result = extractor.adapt_extraction("cost affects revenue")
+        assert 'variables' in result or 'edges' in result
+
+
+class TestTaskDecomposition:
+    """Test task decomposition."""
+    
+    def test_task_analyzer(self):
+        """Test task analyzer."""
+        analyzer = TaskAnalyzer()
+        complexity, should_decompose = analyzer.analyze_task("Analyze the system")
+        assert complexity is not None
+        assert isinstance(should_decompose, bool)
+    
+    def test_task_decomposition(self):
+        """Test task decomposition."""
+        analyzer = TaskAnalyzer()
+        subtasks = analyzer.decompose_task("Analyze price and demand, then compare results")
+        assert len(subtasks) > 0
+        assert all(hasattr(st, 'task_id') for st in subtasks)
+    
+    def test_dependency_graph(self):
+        """Test dependency graph building."""
+        analyzer = TaskAnalyzer()
+        subtasks = analyzer.decompose_task("Analyze X and Y")
+        
+        if len(subtasks) > 1:
+            dependencies = analyzer.build_dependency_graph(subtasks)
+            assert isinstance(dependencies, dict)
+    
+    def test_plan_generation(self):
+        """Test plan generation."""
+        analyzer = TaskAnalyzer()
+        generator = PlanGenerator(analyzer)
+        
+        plan = generator.generate_plan("Analyze the system")
+        assert 'subtasks' in plan
+        assert 'execution_order' in plan
+        assert 'dependencies' in plan
+
+
+class TestExplanationGeneration:
+    """Test explanation generation."""
+    
+    def test_explanation_builder(self):
+        """Test explanation builder."""
+        builder = ExplanationBuilder()
+        
+        # Create mock reasoning chain
+        from schemas.reasoning import ReasoningChain, ReasoningStep
+        chain = ReasoningChain(chain_id="test")
+        step = ReasoningStep(
+            step_id="step1",
+            step_type=StepType.EXTRACTION,
+            operation="extract",
+            input_state={},
+            output_state={},
+            conclusion="test"
+        )
+        chain.add_step(step)
+        
+        explanation = builder.generate_explanation(chain)
+        assert 'steps' in explanation
+        assert 'summary' in explanation
+    
+    def test_transparency_layer(self):
+        """Test transparency layer."""
+        layer = TransparencyLayer()
+        
+        from schemas.reasoning import ReasoningChain, ReasoningStep
+        chain = ReasoningChain(chain_id="test")
+        step = ReasoningStep(
+            step_id="step1",
+            step_type=StepType.EXTRACTION,
+            operation="extract",
+            input_state={},
+            output_state={},
+            conclusion="test",
+            confidence=0.8
+        )
+        chain.add_step(step)
+        
+        trace = layer.show_reasoning_trace(chain)
+        assert 'chain_id' in trace
+        assert 'steps' in trace
+        
+        confidence_viz = layer.visualize_confidence(chain)
+        assert 'mean_confidence' in confidence_viz
+
+
+class TestSelfVerification:
+    """Test self-verification and error detection."""
+    
+    def test_consistency_checker(self):
+        """Test consistency checker."""
+        checker = ConsistencyChecker()
+        
+        graph = {
+            'nodes': ['A', 'B', 'C'],
+            'edges': [('A', 'B'), ('B', 'C')]
+        }
+        
+        is_consistent, error = checker.verify_consistency(graph)
+        assert isinstance(is_consistent, bool)
+    
+    def test_epistemic_grounding(self):
+        """Test epistemic grounding verification."""
+        checker = ConsistencyChecker()
+        
+        graph = {
+            'nodes': ['A', 'B', 'C'],
+            'edges': [('A', 'B'), ('B', 'C')]
+        }
+        
+        observables = {'A'}
+        all_grounded, ungrounded = checker.verify_epistemic_grounding(graph, observables)
+        assert isinstance(all_grounded, bool)
+        assert isinstance(ungrounded, list)
+    
+    def test_error_detector(self):
+        """Test error detection."""
+        detector = ErrorDetector()
+        
+        from schemas.reasoning import ReasoningChain, ReasoningStep
+        chain = ReasoningChain(chain_id="test")
+        step = ReasoningStep(
+            step_id="step1",
+            step_type=StepType.EXTRACTION,
+            operation="extract",
+            input_state={},
+            output_state={},
+            conclusion="test",
+            confidence=0.3  # Low confidence
+        )
+        chain.add_step(step)
+        
+        graph = {'nodes': ['A'], 'edges': []}
+        errors = detector.detect_errors(chain, graph)
+        assert isinstance(errors, list)
+
+
+class TestConsistencyEngine:
+    """Test consistency engine."""
+    
+    def test_deterministic_processor(self):
+        """Test deterministic processing."""
+        processor = DeterministicProcessor(seed=42)
+        
+        # Get random number
+        r1 = processor.get_random()
+        r2 = processor.get_random()
+        
+        # Reset seed and get again
+        processor.reset_seed(42)
+        r3 = processor.get_random()
+        
+        # Should be deterministic (same seed -> same sequence)
+        assert r1 == r3
+    
+    def test_state_snapshot(self):
+        """Test state snapshot."""
+        from architecture.hybrid.consistency_engine import StateSnapshot
+        snapshot_manager = StateSnapshot()
+        
+        state = {'nodes': ['A', 'B'], 'edges': [('A', 'B')]}
+        snapshot_id = snapshot_manager.snapshot(state)
+        
+        assert snapshot_id is not None
+        retrieved = snapshot_manager.get_snapshot(snapshot_id)
+        assert retrieved == state
+    
+    def test_consistency_engine(self):
+        """Test consistency engine."""
+        engine = ConsistencyEngine(seed=42)
+        
+        initial_state = {'value': 0}
+        operations = [lambda s, rng: {'value': s['value'] + 1}]
+        
+        result, snapshot_ids = engine.process_with_snapshots(initial_state, operations)
+        assert result['value'] == 1
+        assert len(snapshot_ids) > 0
+
+
+class TestCausalValidation:
+    """Test causal validation."""
+    
+    def test_causal_relationship_validation(self):
+        """Test causal relationship validation."""
+        from architecture.hybrid.hybrid_agent import SymbolicReasoner
+        from templates.graph_management import GraphManager
+        
+        graph_manager = GraphManager()
+        reasoner = SymbolicReasoner(graph_manager)
+        
+        graph = {
+            'nodes': ['A', 'B', 'C'],
+            'edges': [('A', 'B')]
+        }
+        
+        is_valid, error = reasoner.validate_causal_relationship('A', 'B', graph)
+        assert isinstance(is_valid, bool)
+
+
+class TestIntegration:
+    """Integration tests for full hybrid agent."""
+    
+    def test_hybrid_agent_creation(self):
+        """Test creating hybrid agent with all features enabled."""
+        agent = HybridAgent(
+            enable_conversation=True,
+            enable_reasoning_tracking=True,
+            enable_few_shot_learning=True,
+            enable_task_decomposition=True,
+            enable_explanations=True,
+            enable_verification=True,
+            enable_consistency=True
+        )
+        
+        assert agent.conversation_history is not None
+        assert agent.reasoning_tracker is not None
+        assert agent.example_store is not None
+        assert agent.explanation_builder is not None
+        assert agent.consistency_checker is not None
+        assert agent.consistency_engine is not None
+    
+    def test_simple_task(self):
+        """Test simple task execution."""
+        agent = HybridAgent()
+        response = agent.run("price depends on demand")
+        assert isinstance(response, str)
+        assert len(response) > 0
+    
+    def test_conversation_context(self):
+        """Test multi-turn conversation."""
+        agent = HybridAgent(enable_conversation=True)
+        
+        response1 = agent.run("What affects price?")
+        context = agent.conversation_history.context
+        
+        response2 = agent.run("How about demand?", context=context)
+        assert isinstance(response2, str)
+        assert len(agent.conversation_history.context.messages) >= 2
+    
+    def test_chain_of_thought(self):
+        """Test chain-of-thought reasoning."""
+        agent = HybridAgent(enable_reasoning_tracking=True)
+        
+        response = agent.run("price depends on demand", show_reasoning=True)
+        assert isinstance(response, str)
+        
+        # Check if reasoning chain was created
+        if agent.reasoning_tracker and agent.reasoning_tracker.current_chain:
+            assert len(agent.reasoning_tracker.current_chain.steps) > 0
+    
+    def test_few_shot_learning(self):
+        """Test few-shot learning."""
+        agent = HybridAgent(enable_few_shot_learning=True)
+        
+        # Learn from examples
+        examples = [
+            ("price depends on demand", {"variables": ["price", "demand"], "edges": [("demand", "price")]}),
+            ("cost affects profit", {"variables": ["cost", "profit"], "edges": [("cost", "profit")]})
+        ]
+        
+        agent.learn_from_examples(examples)
+        assert len(agent.example_store.examples) == 2
+        
+        # Use learned patterns
+        response = agent.run("quality influences satisfaction")
+        assert isinstance(response, str)
+    
+    def test_scm_parsing(self):
+        """Test JSON SCM parsing."""
+        agent = HybridAgent()
+        
+        scm_task = """
+        {
+          "task_id": "test",
+          "variables": [
+            { "id": "S", "role": "state", "domain": "real" },
+            { "id": "C", "role": "state", "domain": "real" }
+          ],
+          "equations": [
+            {
+              "id": "S_next",
+              "defines": "S[t+1]",
+              "parents": ["S[t]", "C[t]"],
+              "expr": "S[t] + C[t]"
+            }
+          ]
+        }
+        Parse this SCM.
+        """
+        
+        response = agent.run(scm_task)
+        assert isinstance(response, str)
+        # Should parse SCM successfully
+        assert "error" not in response.lower() or "epistemic" not in response.lower()
+
+
+class TestDeterministicOperations:
+    """Test deterministic operations."""
+    
+    def test_deterministic_reproducibility(self):
+        """Test that operations are reproducible."""
+        agent1 = HybridAgent(seed=42, enable_consistency=True)
+        agent2 = HybridAgent(seed=42, enable_consistency=True)
+        
+        response1 = agent1.run("price depends on demand")
+        response2 = agent2.run("price depends on demand")
+        
+        # Should be identical with same seed
+        assert response1 == response2
+    
+    def test_consistency_engine_integration(self):
+        """Test consistency engine integration."""
+        agent = HybridAgent(enable_consistency=True, seed=42)
+        
+        # Process with snapshots
+        if agent.consistency_engine:
+            initial_state = {'test': 0}
+            operations = [lambda s, rng: {'test': s['test'] + 1}]
+            
+            result, snapshots = agent.consistency_engine.process_with_snapshots(initial_state, operations)
+            assert result['test'] == 1
+            assert len(snapshots) > 0
+
+
+class TestErrorHandling:
+    """Test error handling and self-correction."""
+    
+    def test_error_detection(self):
+        """Test error detection."""
+        agent = HybridAgent(enable_verification=True)
+        
+        # Run with potentially problematic input
+        response = agent.run("identify past policy")
+        assert isinstance(response, str)
+        # Should detect epistemic issues
+        assert "epistemic" in response.lower() or len(response) > 0
+    
+    def test_self_correction(self):
+        """Test self-correction."""
+        agent = HybridAgent(enable_verification=True)
+        
+        if agent.self_corrector:
+            errors = [{'type': 'low_confidence', 'step_id': 'test'}]
+            graph = {'nodes': [], 'edges': []}
+            
+            corrections = agent.self_corrector.correct_errors(errors, graph)
+            assert isinstance(corrections, list)
+
+
+if __name__ == "__main__":
+    pytest.main([__file__, "-v"])