agent-os-kernel 1.1.0__py3-none-any.whl → 1.2.0__py3-none-any.whl

modules/cmvk/tests/test_verification.py

@@ -0,0 +1,255 @@
+"""
+Unit tests for CMVK verification module.
+
+Tests pure verification functions.
+"""
+
+import numpy as np
+import pytest
+
+from cmvk import (
+    DriftType,
+    VerificationScore,
+    aggregate_scores,
+    verify,
+    verify_batch,
+    verify_distributions,
+    verify_embeddings,
+    verify_sequences,
+)
+
+
+class TestVerify:
+    """Tests for the main verify() function."""
+
+    def test_identical_outputs_have_zero_drift(self):
+        """Identical outputs should have zero drift."""
+        text = "def add(a, b): return a + b"
+        score = verify(text, text)
+        assert score.drift_score == 0.0
+        assert score.confidence > 0.8
+
+    def test_empty_outputs_have_zero_drift(self):
+        """Two empty outputs should have zero drift."""
+        score = verify("", "")
+        assert score.drift_score == 0.0
+        assert score.confidence == 1.0
+
+    def test_one_empty_output_has_full_drift(self):
+        """One empty, one non-empty should have full drift."""
+        score = verify("some content", "")
+        assert score.drift_score == 1.0
+
+        score = verify("", "some content")
+        assert score.drift_score == 1.0
+
+    def test_similar_outputs_have_low_drift(self):
+        """Similar outputs should have low drift."""
+        a = "def add(a, b): return a + b"
+        b = "def add(x, y): return x + y"
+        score = verify(a, b)
+        assert score.drift_score < 0.5
+        assert score.confidence > 0.5
+
+    def test_different_outputs_have_high_drift(self):
+        """Very different outputs should have higher drift than similar ones."""
+        similar_a = "def add(a, b): return a + b"
+        similar_b = "def add(x, y): return x + y"
+        different_a = "def add(a, b): return a + b"
+        different_b = "The quick brown fox jumps over the lazy dog"
+
+        similar_score = verify(similar_a, similar_b)
+        different_score = verify(different_a, different_b)
+
+        # Different outputs should have higher drift than similar ones
+        assert different_score.drift_score > similar_score.drift_score
+        # And should have some meaningful drift
+        assert different_score.drift_score > 0.3
+
+    def test_returns_verification_score(self):
+        """verify() should return a VerificationScore."""
+        score = verify("hello", "world")
+        assert isinstance(score, VerificationScore)
+        assert isinstance(score.drift_score, float)
+        assert isinstance(score.confidence, float)
+        assert isinstance(score.drift_type, DriftType)
+        assert isinstance(score.details, dict)
+
+    def test_drift_score_is_bounded(self):
+        """Drift score should be between 0 and 1."""
+        test_cases = [
+            ("", ""),
+            ("a", "b"),
+            ("hello world", "goodbye world"),
+            ("x" * 1000, "y" * 1000),
+        ]
+        for a, b in test_cases:
+            score = verify(a, b)
+            assert 0.0 <= score.drift_score <= 1.0
+            assert 0.0 <= score.confidence <= 1.0
+
+    def test_verification_score_is_immutable(self):
+        """VerificationScore should be frozen/immutable."""
+        score = verify("a", "b")
+        with pytest.raises(AttributeError):
+            score.drift_score = 0.5
+
+
+class TestVerifyEmbeddings:
+    """Tests for verify_embeddings()."""
+
+    def test_identical_embeddings_have_zero_drift(self):
+        """Identical embeddings should have zero drift."""
+        emb = np.array([0.1, 0.2, 0.3, 0.4])
+        score = verify_embeddings(emb, emb)
+        assert score.drift_score < 0.01
+
+    def test_orthogonal_embeddings_have_high_drift(self):
+        """Orthogonal embeddings should have high drift."""
+        emb_a = np.array([1.0, 0.0, 0.0, 0.0])
+        emb_b = np.array([0.0, 1.0, 0.0, 0.0])
+        score = verify_embeddings(emb_a, emb_b)
+        assert score.drift_score >= 0.5  # Orthogonal = cosine distance 1.0, normalized to 0.5
+
+    def test_opposite_embeddings_have_maximum_drift(self):
+        """Opposite embeddings should have maximum drift."""
+        emb_a = np.array([1.0, 1.0, 1.0, 1.0])
+        emb_b = np.array([-1.0, -1.0, -1.0, -1.0])
+        score = verify_embeddings(emb_a, emb_b)
+        assert score.drift_score > 0.9
+
+    def test_shape_mismatch_returns_high_drift(self):
+        """Different shaped embeddings should return high drift."""
+        emb_a = np.array([0.1, 0.2, 0.3])
+        emb_b = np.array([0.1, 0.2, 0.3, 0.4])
+        score = verify_embeddings(emb_a, emb_b)
+        assert score.drift_score == 1.0
+        assert "shape_mismatch" in score.details.get("reason", "")
+
+
+class TestVerifyDistributions:
+    """Tests for verify_distributions()."""
+
+    def test_identical_distributions_have_zero_drift(self):
+        """Identical distributions should have zero drift."""
+        dist = np.array([0.2, 0.3, 0.5])
+        score = verify_distributions(dist, dist)
+        assert score.drift_score < 0.01
+
+    def test_different_distributions_have_positive_drift(self):
+        """Different distributions should have positive drift."""
+        dist_a = np.array([0.9, 0.05, 0.05])
+        dist_b = np.array([0.05, 0.05, 0.9])
+        score = verify_distributions(dist_a, dist_b)
+        assert score.drift_score > 0.5
+
+    def test_details_include_divergence_metrics(self):
+        """Details should include KL and JS divergence."""
+        dist_a = np.array([0.5, 0.5])
+        dist_b = np.array([0.7, 0.3])
+        score = verify_distributions(dist_a, dist_b)
+        assert "kl_divergence" in score.details
+        assert "js_divergence" in score.details
+        assert "total_variation" in score.details
+
+
+class TestVerifySequences:
+    """Tests for verify_sequences()."""
+
+    def test_identical_sequences_have_zero_drift(self):
+        """Identical sequences should have zero drift."""
+        seq = ["a", "b", "c", "d"]
+        score = verify_sequences(seq, seq)
+        assert score.drift_score < 0.01
+
+    def test_empty_sequences_have_zero_drift(self):
+        """Two empty sequences should have zero drift."""
+        score = verify_sequences([], [])
+        assert score.drift_score == 0.0
+
+    def test_different_sequences_have_positive_drift(self):
+        """Different sequences should have positive drift."""
+        seq_a = ["a", "b", "c"]
+        seq_b = ["x", "y", "z"]
+        score = verify_sequences(seq_a, seq_b)
+        assert score.drift_score > 0.5
+
+    def test_details_include_edit_distance(self):
+        """Details should include edit distance."""
+        seq_a = ["def", "add", "(", "a", ")"]
+        seq_b = ["def", "add", "(", "x", ")"]
+        score = verify_sequences(seq_a, seq_b)
+        assert "edit_distance" in score.details
+        assert "jaccard_similarity" in score.details
+        assert "lcs_ratio" in score.details
+
+
+class TestBatchOperations:
+    """Tests for batch verification operations."""
+
+    def test_verify_batch_processes_all_pairs(self):
+        """verify_batch should process all pairs."""
+        outputs_a = ["a", "b", "c"]
+        outputs_b = ["x", "y", "z"]
+        scores = verify_batch(outputs_a, outputs_b)
+        assert len(scores) == 3
+        assert all(isinstance(s, VerificationScore) for s in scores)
+
+    def test_verify_batch_raises_on_length_mismatch(self):
+        """verify_batch should raise on mismatched lengths."""
+        outputs_a = ["a", "b"]
+        outputs_b = ["x", "y", "z"]
+        with pytest.raises(ValueError):
+            verify_batch(outputs_a, outputs_b)
+
+    def test_aggregate_scores_computes_statistics(self):
+        """aggregate_scores should compute summary statistics."""
+        outputs_a = ["hello", "world", "test"]
+        outputs_b = ["hello", "earth", "exam"]
+        scores = verify_batch(outputs_a, outputs_b)
+        summary = aggregate_scores(scores)
+
+        assert "count" in summary
+        assert summary["count"] == 3
+        assert "mean_drift" in summary
+        assert "std_drift" in summary
+        assert "min_drift" in summary
+        assert "max_drift" in summary
+        assert "drift_type_distribution" in summary
+
+    def test_aggregate_scores_handles_empty_list(self):
+        """aggregate_scores should handle empty list."""
+        summary = aggregate_scores([])
+        assert summary["count"] == 0
+
+
+class TestPureFunctions:
+    """Tests to ensure functions are pure (no side effects)."""
+
+    def test_verify_is_deterministic(self):
+        """Same inputs should always produce same output."""
+        a, b = "hello world", "hello earth"
+        score1 = verify(a, b)
+        score2 = verify(a, b)
+        assert score1.drift_score == score2.drift_score
+        assert score1.confidence == score2.confidence
+
+    def test_verify_does_not_modify_inputs(self):
+        """verify should not modify input strings."""
+        a = "original text"
+        b = "other text"
+        a_copy = a
+        b_copy = b
+        verify(a, b)
+        assert a == a_copy
+        assert b == b_copy
+
+    def test_verify_embeddings_does_not_modify_arrays(self):
+        """verify_embeddings should not modify input arrays."""
+        emb_a = np.array([0.1, 0.2, 0.3])
+        emb_b = np.array([0.4, 0.5, 0.6])
+        original_a = emb_a.copy()
+        original_b = emb_b.copy()
+        verify_embeddings(emb_a, emb_b)
+        np.testing.assert_array_equal(emb_a, original_a)
+        np.testing.assert_array_equal(emb_b, original_b)

modules/cmvk/tests/unit/__init__.py

@@ -0,0 +1 @@
+"""Unit tests for CMVK."""

modules/cmvk/tests/unit/test_agents.py

@@ -0,0 +1,64 @@
+"""
+Unit tests for the agents module.
+"""
+
+import sys
+from pathlib import Path
+
+import pytest
+
+# Add src to path
+sys.path.insert(0, str(Path(__file__).parent.parent / "src"))
+
+from cross_model_verification_kernel.agents.base_agent import BaseAgent
+from cross_model_verification_kernel.core.types import GenerationResult, VerificationResult
+
+
+class MockGenerator(BaseAgent):
+    """Mock generator for testing."""
+
+    def generate(self, task, context=None):
+        return GenerationResult(
+            solution="mock solution", explanation="mock explanation", test_cases="mock tests"
+        )
+
+    def verify(self, context):
+        raise NotImplementedError()
+
+
+class MockVerifier(BaseAgent):
+    """Mock verifier for testing."""
+
+    def generate(self, task, context=None):
+        raise NotImplementedError()
+
+    def verify(self, context):
+        from cross_model_verification_kernel.core.types import VerificationOutcome
+
+        return VerificationResult(
+            outcome=VerificationOutcome.PASS, confidence=0.9, reasoning="mock verification"
+        )
+
+
+class TestBaseAgent:
+    """Tests for BaseAgent interface."""
+
+    def test_mock_generator(self):
+        """Test mock generator implementation."""
+        agent = MockGenerator("test-model", "test-key")
+        result = agent.generate("test task")
+
+        assert isinstance(result, GenerationResult)
+        assert result.solution == "mock solution"
+
+    def test_mock_verifier(self):
+        """Test mock verifier implementation."""
+        agent = MockVerifier("test-model", "test-key")
+        result = agent.verify({"task": "test"})
+
+        assert isinstance(result, VerificationResult)
+        assert result.confidence == 0.9
+
+
+if __name__ == "__main__":
+    pytest.main([__file__, "-v"])

modules/cmvk/tests/unit/test_cli.py

@@ -0,0 +1,224 @@
+"""
+Tests for the CLI module.
+"""
+
+import json
+import sys
+from pathlib import Path
+from unittest.mock import MagicMock, patch
+
+import pytest
+from typer.testing import CliRunner
+
+sys.path.insert(0, str(Path(__file__).parent.parent / "src"))
+
+from cross_model_verification_kernel.cli import (
+    GeneratorModel,
+    OutputFormat,
+    VerifierModel,
+    app,
+    set_seed,
+)
+
+runner = CliRunner()
+
+
+class TestCLIBasics:
+    """Basic CLI functionality tests."""
+
+    def test_version(self):
+        """Test --version flag."""
+        result = runner.invoke(app, ["--version"])
+        assert result.exit_code == 0
+        assert "1.0.0" in result.stdout
+        assert "CMVK" in result.stdout
+
+    def test_help(self):
+        """Test --help flag."""
+        result = runner.invoke(app, ["--help"])
+        assert result.exit_code == 0
+        assert "Cross-Model Verification Kernel" in result.stdout
+        assert "run" in result.stdout
+        assert "config" in result.stdout
+
+    def test_run_help(self):
+        """Test run command help."""
+        result = runner.invoke(app, ["run", "--help"])
+        assert result.exit_code == 0
+        assert "--task" in result.stdout
+        assert "--generator" in result.stdout
+        assert "--verifier" in result.stdout
+        assert "--seed" in result.stdout
+
+    def test_models_command(self):
+        """Test models command lists available models."""
+        result = runner.invoke(app, ["models"])
+        assert result.exit_code == 0
+        assert "gpt-4o" in result.stdout
+        assert "gemini-1.5-pro" in result.stdout
+        assert "claude-3-5-sonnet" in result.stdout
+
+
+class TestCLIConfig:
+    """Tests for config command."""
+
+    def test_config_no_args(self):
+        """Test config command with no arguments."""
+        result = runner.invoke(app, ["config"])
+        assert result.exit_code == 0
+        assert "--show" in result.stdout or "--init" in result.stdout
+
+    def test_config_show_missing_file(self):
+        """Test config --show with missing file."""
+        result = runner.invoke(app, ["config", "--show", "--path", "nonexistent.yaml"])
+        assert "not found" in result.stdout.lower()
+
+    def test_config_init(self, tmp_path):
+        """Test config --init creates a config file."""
+        config_path = tmp_path / "test_config.yaml"
+        result = runner.invoke(app, ["config", "--init", "--path", str(config_path)])
+
+        assert result.exit_code == 0
+        assert config_path.exists()
+
+        content = config_path.read_text()
+        assert "api_keys" in content
+        assert "kernel" in content
+        assert "max_loops" in content
+
+
+class TestCLIRun:
+    """Tests for run command."""
+
+    def test_run_missing_api_key(self):
+        """Test run command fails without API key."""
+        with patch.dict("os.environ", {}, clear=True):
+            result = runner.invoke(app, ["run", "--task", "test task"])
+            assert result.exit_code == 1
+            assert "OPENAI_API_KEY" in result.stdout
+
+    @patch("cross_model_verification_kernel.cli.OpenAIGenerator")
+    @patch("cross_model_verification_kernel.cli.GeminiVerifier")
+    @patch("cross_model_verification_kernel.cli.VerificationKernel")
+    def test_run_with_mocked_kernel(self, mock_kernel_class, mock_verifier, mock_generator):
+        """Test run command with mocked components."""
+        # Setup mocks
+        mock_state = MagicMock()
+        mock_state.is_complete = True
+        mock_state.final_result = "def solution(): pass"
+        mock_state.current_loop = 2
+        mock_state.verification_history = []
+
+        mock_kernel = MagicMock()
+        mock_kernel.execute.return_value = mock_state
+        mock_kernel_class.return_value = mock_kernel
+
+        with patch.dict("os.environ", {"OPENAI_API_KEY": "test-key", "GOOGLE_API_KEY": "test-key"}):
+            result = runner.invoke(app, ["run", "--task", "Write hello world", "--output", "json"])
+
+            # Should succeed or fail gracefully
+            if result.exit_code == 0:
+                output = json.loads(result.stdout)
+                assert "success" in output
+
+    def test_run_invalid_generator(self):
+        """Test run with invalid generator model."""
+        with patch.dict("os.environ", {"OPENAI_API_KEY": "test"}):
+            result = runner.invoke(app, ["run", "--task", "test", "--generator", "invalid-model"])
+            assert result.exit_code != 0
+
+
+class TestCLIVisualize:
+    """Tests for visualize command."""
+
+    def test_visualize_missing_file(self):
+        """Test visualize with missing trace file."""
+        result = runner.invoke(app, ["visualize", "nonexistent.json"])
+        assert result.exit_code == 1
+        assert "not found" in result.stdout.lower()
+
+    def test_visualize_valid_trace(self, tmp_path):
+        """Test visualize with a valid trace file."""
+        trace_file = tmp_path / "test_trace.json"
+        trace_data = {
+            "task_id": "test_001",
+            "task_description": "Test task description",
+            "total_loops": 3,
+            "success": True,
+            "generator_model": "gpt-4o",
+            "verifier_model": "gemini-1.5-pro",
+            "conversation_trace": [
+                {"type": "generation", "loop": 1},
+                {"type": "verification", "loop": 1},
+            ],
+        }
+        trace_file.write_text(json.dumps(trace_data))
+
+        result = runner.invoke(app, ["visualize", str(trace_file)])
+
+        assert result.exit_code == 0
+        assert "test_001" in result.stdout
+        assert "gpt-4o" in result.stdout
+
+
+class TestCLIBenchmark:
+    """Tests for benchmark command."""
+
+    def test_benchmark_missing_dataset(self):
+        """Test benchmark with missing dataset."""
+        result = runner.invoke(app, ["benchmark", "--dataset", "nonexistent_dataset"])
+        assert result.exit_code == 1
+        assert "not found" in result.stdout.lower()
+
+
+class TestSetSeed:
+    """Tests for seed functionality."""
+
+    def test_set_seed_deterministic(self):
+        """Test that set_seed produces deterministic results."""
+        import random
+
+        set_seed(42)
+        values1 = [random.random() for _ in range(5)]
+
+        set_seed(42)
+        values2 = [random.random() for _ in range(5)]
+
+        assert values1 == values2
+
+    def test_set_seed_different_seeds(self):
+        """Test that different seeds produce different results."""
+        import random
+
+        set_seed(42)
+        values1 = [random.random() for _ in range(5)]
+
+        set_seed(123)
+        values2 = [random.random() for _ in range(5)]
+
+        assert values1 != values2
+
+
+class TestEnums:
+    """Tests for CLI enums."""
+
+    def test_generator_models(self):
+        """Test generator model enum values."""
+        assert GeneratorModel.GPT_4O.value == "gpt-4o"
+        assert GeneratorModel.O1.value == "o1"
+        assert GeneratorModel.O3_MINI.value == "o3-mini"
+
+    def test_verifier_models(self):
+        """Test verifier model enum values."""
+        assert VerifierModel.GEMINI_15_PRO.value == "gemini-1.5-pro"
+        assert VerifierModel.CLAUDE_35_SONNET.value == "claude-3-5-sonnet-20241022"
+
+    def test_output_formats(self):
+        """Test output format enum values."""
+        assert OutputFormat.PRETTY.value == "pretty"
+        assert OutputFormat.JSON.value == "json"
+        assert OutputFormat.MINIMAL.value == "minimal"
+
+
+if __name__ == "__main__":
+    pytest.main([__file__, "-v"])

modules/cmvk/tests/unit/test_core.py

@@ -0,0 +1,126 @@
+"""
+Unit tests for the Cross-Model Verification Kernel.
+"""
+
+import sys
+from pathlib import Path
+
+import pytest
+
+# Add src to path
+sys.path.insert(0, str(Path(__file__).parent.parent / "src"))
+
+from cross_model_verification_kernel.core.graph_memory import GraphMemory
+from cross_model_verification_kernel.core.types import (
+    Node,
+    NodeStatus,
+    VerificationOutcome,
+    VerificationResult,
+)
+
+
+class TestGraphMemory:
+    """Tests for the GraphMemory class."""
+
+    def test_create_node(self):
+        """Test node creation."""
+        graph = GraphMemory()
+        node = graph.create_node("test content")
+
+        assert node.id is not None
+        assert node.content == "test content"
+        assert node.status == NodeStatus.PENDING
+        assert node.id in graph.nodes
+
+    def test_update_node_status(self):
+        """Test updating node status."""
+        graph = GraphMemory()
+        node = graph.create_node("test")
+
+        graph.update_node_status(node.id, NodeStatus.VERIFIED)
+        assert graph.nodes[node.id].status == NodeStatus.VERIFIED
+
+    def test_verification_result_addition(self):
+        """Test adding verification results to nodes."""
+        graph = GraphMemory()
+        node = graph.create_node("test")
+
+        result = VerificationResult(outcome=VerificationOutcome.PASS, confidence=0.9)
+
+        graph.add_verification_result(node.id, result)
+        assert len(graph.nodes[node.id].verification_results) == 1
+        assert graph.nodes[node.id].status == NodeStatus.VERIFIED
+
+    def test_loop_detection(self):
+        """Test state loop detection."""
+        graph = GraphMemory()
+
+        state_hash = GraphMemory.generate_state_hash("task", "solution", 1)
+        assert not graph.has_visited_state(state_hash)
+
+        graph.mark_state_visited(state_hash)
+        assert graph.has_visited_state(state_hash)
+
+    def test_solution_caching(self):
+        """Test caching of verified solutions."""
+        graph = GraphMemory()
+
+        problem_hash = "test_problem"
+        solution = "test_solution"
+
+        assert graph.get_cached_solution(problem_hash) is None
+
+        graph.cache_solution(problem_hash, solution)
+        assert graph.get_cached_solution(problem_hash) == solution
+
+    def test_get_verified_nodes(self):
+        """Test retrieving verified nodes."""
+        graph = GraphMemory()
+
+        node1 = graph.create_node("verified")
+        graph.update_node_status(node1.id, NodeStatus.VERIFIED)
+
+        node2 = graph.create_node("pending")
+
+        verified = graph.get_verified_nodes()
+        assert len(verified) == 1
+        assert verified[0].id == node1.id
+
+    def test_clear(self):
+        """Test clearing graph state."""
+        graph = GraphMemory()
+        graph.create_node("test")
+        graph.mark_state_visited("state1")
+        graph.cache_solution("prob1", "sol1")
+
+        graph.clear()
+
+        assert len(graph.nodes) == 0
+        assert len(graph.visited_states) == 0
+        assert len(graph.verified_cache) == 0
+
+
+class TestTypes:
+    """Tests for data types."""
+
+    def test_verification_result_has_critical_issues(self):
+        """Test detection of critical issues."""
+        result1 = VerificationResult(
+            outcome=VerificationOutcome.FAIL, confidence=0.8, critical_issues=["Bug found"]
+        )
+        assert result1.has_critical_issues()
+
+        result2 = VerificationResult(outcome=VerificationOutcome.PASS, confidence=0.9)
+        assert not result2.has_critical_issues()
+
+    def test_node_is_verified(self):
+        """Test node verification status check."""
+        node = Node(id="test", content="test", status=NodeStatus.VERIFIED)
+        assert node.is_verified()
+
+        node.status = NodeStatus.PENDING
+        assert not node.is_verified()
+
+
+if __name__ == "__main__":
+    pytest.main([__file__, "-v"])