synth-ai 0.1.9__py3-none-any.whl → 0.2.1.dev0__py3-none-any.whl

synth_ai/environments/examples/nethack/units/test_nethack_environment.py

@@ -0,0 +1,281 @@
+"""Unit tests for NetHack environment."""
+
+import pytest
+import asyncio
+from uuid import uuid4
+
+from synth_ai.environments.environment.tools import EnvToolCall
+from synth_ai.environments.tasks.core import Impetus, Intent
+
+from synth_ai.environments.examples.nethack.environment import (
+    NetHackEnvironment,
+    NetHackInteractTool,
+)
+from synth_ai.environments.examples.nethack.taskset import (
+    NetHackTaskInstanceMetadata,
+    NetHackTaskInstance,
+)
+
+
+class TestNetHackEnvironment:
+    """Test cases for NetHack environment."""
+
+    @pytest.fixture
+    def mock_task_instance(self):
+        """Create a mock task instance for testing."""
+        metadata = NetHackTaskInstanceMetadata(
+            character_role="knight",
+            starting_level=1,
+            target_depth=3,
+            time_limit=500,
+            difficulty="beginner",
+            special_objectives=["Defeat 10 monsters"],
+            seed=123,
+        )
+
+        return NetHackTaskInstance(
+            id=uuid4(),
+            impetus=Impetus(instructions="Test knight adventure"),
+            intent=Intent(
+                rubric={"goal": "Reach depth 3"},
+                gold_trajectories=None,
+                gold_state_diff={},
+            ),
+            metadata=metadata,
+            is_reproducible=True,
+            initial_engine_snapshot=None,
+        )
+
+    @pytest.mark.asyncio
+    async def test_environment_initialization(self, mock_task_instance):
+        """Test environment initialization."""
+        env = NetHackEnvironment(mock_task_instance)
+
+        assert env.name == "NetHack"
+        assert env.task_instance == mock_task_instance
+        assert env.engine is not None
+
+        # Initialize and check observation
+        obs = await env.initialize()
+
+        assert isinstance(obs, dict)
+        assert "ascii_map" in obs
+        assert "message" in obs
+        assert "character_stats" in obs
+        assert "terminated" in obs
+        assert obs["terminated"] is False
+
+    @pytest.mark.asyncio
+    async def test_step_with_valid_action(self, mock_task_instance):
+        """Test stepping with valid actions."""
+        env = NetHackEnvironment(mock_task_instance)
+        await env.initialize()
+
+        # Test simple string action
+        obs = await env.step("north")
+        assert "last_action" in obs
+        assert obs["last_action"] == "north"
+        assert obs["turn_count"] == 1
+
+        # Test another movement
+        obs = await env.step("east")
+        assert obs["last_action"] == "east"
+        assert obs["turn_count"] == 2
+
+    @pytest.mark.asyncio
+    async def test_step_with_invalid_action(self, mock_task_instance):
+        """Test stepping with invalid actions."""
+        env = NetHackEnvironment(mock_task_instance)
+        await env.initialize()
+
+        # Test invalid action
+        obs = await env.step("invalid_action_xyz")
+        assert "error" in obs
+        assert "Unknown action" in obs["error"]
+
+    @pytest.mark.asyncio
+    async def test_tool_call_formats(self, mock_task_instance):
+        """Test various tool call input formats."""
+        env = NetHackEnvironment(mock_task_instance)
+        await env.initialize()
+
+        # Test dict with action key
+        obs = await env.step({"action": "wait"})
+        assert obs["last_action"] == "wait"
+
+        # Test EnvToolCall format
+        tool_call = EnvToolCall(tool="interact", args={"action": "search"})
+        obs = await env.step(tool_call)
+        assert obs["last_action"] == "search"
+
+        # Test list format
+        obs = await env.step([{"action": "inventory"}])
+        assert obs["last_action"] == "inventory"
+
+        # Test nested tool_calls format
+        obs = await env.step({"tool_calls": [{"args": {"action": "look"}}]})
+        assert obs["last_action"] == "look"
+
+    @pytest.mark.asyncio
+    async def test_checkpoint(self, mock_task_instance):
+        """Test checkpoint functionality."""
+        env = NetHackEnvironment(mock_task_instance)
+        await env.initialize()
+
+        # Take some actions
+        await env.step("north")
+        await env.step("east")
+
+        # Create checkpoint
+        checkpoint_obs = await env.checkpoint()
+
+        assert "final_score" in checkpoint_obs
+        assert "max_depth" in checkpoint_obs
+        assert "turn_count_final" in checkpoint_obs
+        assert "total_reward" in checkpoint_obs
+
+    @pytest.mark.asyncio
+    async def test_terminate(self, mock_task_instance):
+        """Test environment termination."""
+        env = NetHackEnvironment(mock_task_instance)
+        await env.initialize()
+
+        # Take an action
+        await env.step("wait")
+
+        # Terminate
+        final_obs = await env.terminate()
+
+        assert final_obs["terminated"] is True
+        assert "final_score" in final_obs
+        assert "total_reward" in final_obs
+
+    @pytest.mark.asyncio
+    async def test_validate_tool_calls_edge_cases(self, mock_task_instance):
+        """Test tool call validation edge cases."""
+        env = NetHackEnvironment(mock_task_instance)
+
+        # Test empty list
+        with pytest.raises(ValueError, match="Empty tool calls list"):
+            env.validate_tool_calls([])
+
+        # Test invalid format
+        with pytest.raises(ValueError, match="Invalid tool call format"):
+            env.validate_tool_calls(123)  # type: ignore[arg-type]  # Not a valid format
+
+        # Test nested args
+        call = env.validate_tool_calls({"args": {"action": "north"}})
+        assert call.args["action"] == "north"
+
+        # Test parameters key
+        call = env.validate_tool_calls({"parameters": {"action": "south"}})
+        assert call.args["action"] == "south"
+
+    @pytest.mark.asyncio
+    async def test_available_actions(self, mock_task_instance):
+        """Test getting available actions."""
+        env = NetHackEnvironment(mock_task_instance)
+
+        actions = env.get_available_actions()
+        assert isinstance(actions, list)
+        assert "north" in actions
+        assert "inventory" in actions
+        assert "a" in actions  # Menu action
+
+        descriptions = env.get_action_descriptions()
+        assert isinstance(descriptions, dict)
+        assert descriptions["north"] == "move north"
+        assert descriptions["inventory"] == "check inventory"
+
+
+class TestNetHackInteractTool:
+    """Test cases for NetHack interact tool."""
+
+    @pytest.fixture
+    def mock_task_instance(self):
+        """Create a mock task instance for testing."""
+        metadata = NetHackTaskInstanceMetadata(
+            character_role="knight",
+            starting_level=1,
+            target_depth=3,
+            time_limit=500,
+            difficulty="beginner",
+            special_objectives=["Defeat 10 monsters"],
+            seed=123,
+        )
+
+        return NetHackTaskInstance(
+            id=uuid4(),
+            impetus=Impetus(instructions="Test knight adventure"),
+            intent=Intent(
+                rubric={"goal": "Test objectives"},
+                gold_trajectories=None,
+                gold_state_diff={},
+            ),
+            metadata=metadata,
+            is_reproducible=True,
+            initial_engine_snapshot=None,
+        )
+
+    @pytest.fixture
+    def mock_engine(self, mock_task_instance):
+        """Create a mock engine for testing."""
+        from synth_ai.environments.examples.nethack.engine import NetHackEngine
+
+        return NetHackEngine(mock_task_instance)
+
+    @pytest.mark.asyncio
+    async def test_interact_tool_valid_action(self, mock_engine):
+        """Test interact tool with valid action."""
+        await mock_engine._reset_engine()
+        tool = NetHackInteractTool(mock_engine)
+
+        call = EnvToolCall(tool="interact", args={"action": "wait"})
+        result = await tool(call)
+
+        assert result.ok is True
+        assert "public_state" in result.payload
+        assert "private_state" in result.payload
+        assert result.payload["public_state"].last_action == "wait"
+
+    @pytest.mark.asyncio
+    async def test_interact_tool_no_action(self, mock_engine):
+        """Test interact tool with missing action."""
+        await mock_engine._reset_engine()
+        tool = NetHackInteractTool(mock_engine)
+
+        call = EnvToolCall(tool="interact", args={})
+        result = await tool(call)
+
+        assert result.ok is False
+        # KeyError is caught and returned as string
+        assert "'action'" in result.error
+
+    @pytest.mark.asyncio
+    async def test_interact_tool_invalid_action(self, mock_engine):
+        """Test interact tool with invalid action."""
+        await mock_engine._reset_engine()
+        tool = NetHackInteractTool(mock_engine)
+
+        call = EnvToolCall(tool="interact", args={"action": "fly"})
+        result = await tool(call)
+
+        assert result.ok is False
+        assert "Unknown action" in result.error
+
+    @pytest.mark.asyncio
+    async def test_interact_tool_game_over_validation(self, mock_engine):
+        """Test interact tool validation when game is over."""
+        await mock_engine._reset_engine()
+        tool = NetHackInteractTool(mock_engine)
+
+        # Manually terminate the game
+        mock_engine.public_state.terminated = True
+        mock_engine.private_state.terminated = True
+
+        # Try non-quit action
+        call = EnvToolCall(tool="interact", args={"action": "north"})
+        result = await tool(call)
+
+        assert result.ok is False
+        assert "Game is over" in result.error

synth_ai/environments/examples/nethack/units/test_nethack_taskset.py

@@ -0,0 +1,213 @@
+"""Unit tests for NetHack taskset."""
+
+import pytest
+import asyncio
+
+from synth_ai.environments.examples.nethack.taskset import (
+    create_nethack_taskset,
+    NetHackTaskInstance,
+    NetHackTaskInstanceMetadata,
+    CHARACTER_ROLES,
+    SPECIAL_OBJECTIVES,
+)
+
+
+class TestNetHackTaskSet:
+    """Test cases for NetHack taskset generation."""
+
+    @pytest.mark.asyncio
+    async def test_taskset_creation(self):
+        """Test basic taskset creation."""
+        taskset = await create_nethack_taskset()
+
+        assert taskset.name == "NetHack TaskSet"
+        assert len(taskset.instances) == 100  # Sum of all difficulty counts
+        assert taskset.split_info._is_split_defined is True
+
+    @pytest.mark.asyncio
+    async def test_task_instance_properties(self):
+        """Test properties of generated task instances."""
+        taskset = await create_nethack_taskset()
+
+        for instance in taskset.instances[:10]:  # Check first 10
+            assert isinstance(instance, NetHackTaskInstance)
+            assert isinstance(instance.metadata, NetHackTaskInstanceMetadata)
+
+            # Check required fields
+            assert instance.id is not None
+            assert instance.impetus.instructions != ""
+            assert instance.intent.rubric is not None
+            assert instance.is_reproducible is True
+
+            # Check metadata
+            meta = instance.metadata
+            assert meta.character_role in CHARACTER_ROLES
+            assert meta.starting_level == 1
+            assert meta.target_depth > 0
+            assert meta.time_limit > 0
+            assert meta.difficulty in [
+                "tutorial",
+                "beginner",
+                "intermediate",
+                "advanced",
+                "expert",
+            ]
+            assert isinstance(meta.special_objectives, list)
+            assert meta.seed is not None
+
+    @pytest.mark.asyncio
+    async def test_difficulty_distribution(self):
+        """Test that difficulties are properly distributed."""
+        taskset = await create_nethack_taskset()
+
+        difficulty_counts = {
+            "tutorial": 0,
+            "beginner": 0,
+            "intermediate": 0,
+            "advanced": 0,
+            "expert": 0,
+        }
+
+        for instance in taskset.instances:
+            difficulty_counts[instance.metadata.difficulty] += 1
+
+        assert difficulty_counts["tutorial"] == 20
+        assert difficulty_counts["beginner"] == 30
+        assert difficulty_counts["intermediate"] == 25
+        assert difficulty_counts["advanced"] == 15
+        assert difficulty_counts["expert"] == 10
+
+    @pytest.mark.asyncio
+    async def test_character_role_assignment(self):
+        """Test character role assignment by difficulty."""
+        taskset = await create_nethack_taskset()
+
+        # Check tutorial only has tourist
+        tutorial_instances = [i for i in taskset.instances if i.metadata.difficulty == "tutorial"]
+        for inst in tutorial_instances:
+            assert inst.metadata.character_role == "tourist"
+
+        # Check expert has all roles
+        expert_instances = [i for i in taskset.instances if i.metadata.difficulty == "expert"]
+        expert_roles = set(inst.metadata.character_role for inst in expert_instances)
+        assert len(expert_roles) > 1  # Should have multiple roles
+
+    @pytest.mark.asyncio
+    async def test_objective_assignment(self):
+        """Test special objectives assignment."""
+        taskset = await create_nethack_taskset()
+
+        # Check objectives are from valid categories
+        all_valid_objectives = []
+        for category in SPECIAL_OBJECTIVES.values():
+            all_valid_objectives.extend(category)
+
+        for instance in taskset.instances:
+            for obj in instance.metadata.special_objectives:
+                assert obj in all_valid_objectives
+
+        # Check objective count by difficulty
+        tutorial_inst = next(i for i in taskset.instances if i.metadata.difficulty == "tutorial")
+        assert len(tutorial_inst.metadata.special_objectives) == 1
+
+        expert_inst = next(i for i in taskset.instances if i.metadata.difficulty == "expert")
+        assert len(expert_inst.metadata.special_objectives) == 4
+
+    @pytest.mark.asyncio
+    async def test_instruction_content(self):
+        """Test that instructions contain necessary information."""
+        taskset = await create_nethack_taskset()
+
+        for instance in taskset.instances[:5]:  # Check first 5
+            instructions = instance.impetus.instructions
+
+            # Check key elements are present
+            assert instance.metadata.character_role in instructions
+            assert str(instance.metadata.target_depth) in instructions
+            assert str(instance.metadata.time_limit) in instructions
+            assert "Additional objectives:" in instructions
+            assert "Character strengths:" in instructions
+            assert "Character weaknesses:" in instructions
+            assert "Tips:" in instructions
+
+    @pytest.mark.asyncio
+    async def test_rubric_structure(self):
+        """Test intent rubric structure."""
+        taskset = await create_nethack_taskset()
+
+        for instance in taskset.instances[:5]:
+            rubric = instance.intent.rubric
+
+            assert "goal" in rubric
+            assert "success_criteria" in rubric
+            assert "evaluation_metrics" in rubric
+
+            # Check success criteria
+            assert "primary" in rubric["success_criteria"]
+            assert "secondary" in rubric["success_criteria"]
+
+            # Check evaluation metrics
+            metrics = rubric["evaluation_metrics"]
+            assert metrics["depth_reached"] == instance.metadata.target_depth
+            assert metrics["time_limit"] == instance.metadata.time_limit
+            assert metrics["objectives_completed"] == len(instance.metadata.special_objectives)
+
+    @pytest.mark.asyncio
+    async def test_split_info(self):
+        """Test train/val/test split."""
+        taskset = await create_nethack_taskset()
+
+        total_instances = len(taskset.instances)
+        val_size = len(taskset.split_info.val_instance_ids)
+        test_size = len(taskset.split_info.test_instance_ids)
+
+        # Check split sizes (should be ~10% each)
+        assert val_size == total_instances // 10
+        assert test_size == total_instances // 10
+
+        # Check no overlap
+        assert len(taskset.split_info.val_instance_ids & taskset.split_info.test_instance_ids) == 0
+
+        # Check all split IDs are valid
+        all_ids = {inst.id for inst in taskset.instances}
+        assert taskset.split_info.val_instance_ids.issubset(all_ids)
+        assert taskset.split_info.test_instance_ids.issubset(all_ids)
+
+    @pytest.mark.asyncio
+    async def test_task_serialization(self):
+        """Test task instance serialization."""
+        taskset = await create_nethack_taskset()
+        instance = taskset.instances[0]
+
+        # Serialize
+        serialized = await instance.serialize()
+
+        assert isinstance(serialized, dict)
+        assert "id" in serialized
+        assert "impetus" in serialized
+        assert "intent" in serialized
+        assert "metadata" in serialized
+
+        # Check metadata fields
+        meta = serialized["metadata"]
+        assert meta["character_role"] == instance.metadata.character_role
+        assert meta["target_depth"] == instance.metadata.target_depth
+        assert meta["time_limit"] == instance.metadata.time_limit
+
+        # Deserialize
+        restored = await NetHackTaskInstance.deserialize(serialized)
+
+        assert restored.metadata.character_role == instance.metadata.character_role
+        assert restored.metadata.target_depth == instance.metadata.target_depth
+        assert restored.metadata.time_limit == instance.metadata.time_limit
+        assert restored.metadata.special_objectives == instance.metadata.special_objectives
+
+    @pytest.mark.asyncio
+    async def test_reproducibility(self):
+        """Test that tasks are marked as reproducible."""
+        taskset = await create_nethack_taskset()
+
+        for instance in taskset.instances:
+            assert instance.is_reproducible is True
+            assert instance.metadata.seed is not None
+            assert 0 <= instance.metadata.seed < 2**31