agent-os-kernel 1.1.0__py3-none-any.whl → 1.2.0__py3-none-any.whl

modules/control-plane/tests/test_lifecycle.py

@@ -0,0 +1,1174 @@
+"""
+Tests for Agent Lifecycle Management (v0.2.0)
+
+This module tests the comprehensive lifecycle management features:
+- ACP-001: Agent Health Checks
+- ACP-002: Agent Auto-Recovery
+- ACP-003: Circuit Breaker
+- ACP-004: Agent Scaling
+- ACP-005: Distributed Coordination
+- ACP-006: Agent Dependency Graph
+- ACP-007: Graceful Shutdown
+- ACP-008: Resource Quotas
+- ACP-009: Agent Observability
+- ACP-010: Hot Reload
+"""
+
+import pytest
+import asyncio
+from datetime import datetime, timedelta
+from unittest.mock import Mock, AsyncMock, patch, MagicMock
+import sys
+
+from agent_control_plane.lifecycle import (
+    # Health Monitoring
+    HealthMonitor,
+    HealthCheckConfig,
+    HealthCheckResult,
+    HealthStatus,
+    
+    # Auto-Recovery
+    AutoRecoveryManager,
+    RecoveryConfig,
+    RecoveryEvent,
+    
+    # Circuit Breaker
+    CircuitBreaker,
+    CircuitBreakerConfig,
+    CircuitBreakerOpenError,
+    CircuitState,
+    CircuitBreakerRegistry,
+    
+    # Scaling
+    AgentScaler,
+    ScalingConfig,
+    AgentReplica,
+    AgentState,
+    
+    # Distributed Coordination
+    DistributedCoordinator,
+    LeaderElectionConfig,
+    CoordinationRole,
+    
+    # Dependency Graph
+    DependencyGraph,
+    AgentDependency,
+    
+    # Graceful Shutdown
+    GracefulShutdownManager,
+    ShutdownConfig,
+    ShutdownPhase,
+    
+    # Resource Quotas
+    ResourceQuotaManager,
+    AgentResourceQuota,
+    
+    # Observability
+    AgentObservabilityProvider,
+    
+    # Hot Reload
+    HotReloadManager,
+    HotReloadConfig,
+    
+    # Main Control Plane
+    EnhancedAgentControlPlane,
+    create_control_plane,
+)
+
+
+# ============================================================================
+# Test Fixtures
+# ============================================================================
+
+class MockAgent:
+    """Mock agent for testing"""
+    
+    def __init__(self, name: str = "mock-agent"):
+        self.name = name
+        self.started = False
+        self.stopped = False
+        self._is_alive = True
+        self._is_ready = True
+        self._state = {}
+    
+    async def start(self):
+        self.started = True
+    
+    async def stop(self):
+        self.stopped = True
+    
+    def is_alive(self) -> bool:
+        return self._is_alive
+    
+    def is_ready(self) -> bool:
+        return self._is_ready
+    
+    async def liveness_check(self) -> bool:
+        return self._is_alive
+    
+    async def readiness_check(self) -> bool:
+        return self._is_ready
+    
+    def get_state(self) -> dict:
+        return self._state.copy()
+    
+    def set_state(self, state: dict):
+        self._state = state.copy()
+
+
+@pytest.fixture
+def mock_agent():
+    """Create a mock agent"""
+    return MockAgent()
+
+
+@pytest.fixture
+def health_config():
+    """Create health check config"""
+    return HealthCheckConfig(
+        liveness_interval_seconds=0.1,
+        liveness_timeout_seconds=1.0,
+        liveness_failure_threshold=2,
+        readiness_interval_seconds=0.1,
+        readiness_timeout_seconds=1.0,
+        readiness_failure_threshold=1
+    )
+
+
+@pytest.fixture
+def recovery_config():
+    """Create recovery config"""
+    return RecoveryConfig(
+        enabled=True,
+        max_restarts=3,
+        restart_delay_seconds=0.1,
+        restart_delay_max_seconds=1.0,
+        restart_delay_multiplier=2.0
+    )
+
+
+# ============================================================================
+# ACP-001: Health Check Tests
+# ============================================================================
+
+class TestHealthMonitor:
+    """Tests for health monitoring functionality"""
+    
+    @pytest.mark.asyncio
+    async def test_register_agent(self, health_config):
+        """Test agent registration for health monitoring"""
+        monitor = HealthMonitor(config=health_config)
+        agent = MockAgent("test-agent")
+        
+        monitor.register_agent("test-agent", agent)
+        
+        assert "test-agent" in monitor._agents
+        assert monitor.get_agent_health("test-agent") == HealthStatus.UNKNOWN
+    
+    @pytest.mark.asyncio
+    async def test_liveness_check_success(self, health_config):
+        """Test successful liveness check"""
+        monitor = HealthMonitor(config=health_config)
+        agent = MockAgent()
+        agent._is_alive = True
+        
+        monitor.register_agent("test-agent", agent)
+        result = await monitor._check_liveness("test-agent")
+        
+        assert result.healthy is True
+        assert result.status == HealthStatus.HEALTHY
+    
+    @pytest.mark.asyncio
+    async def test_liveness_check_failure(self, health_config):
+        """Test failed liveness check"""
+        monitor = HealthMonitor(config=health_config)
+        agent = MockAgent()
+        agent._is_alive = False
+        
+        monitor.register_agent("test-agent", agent)
+        result = await monitor._check_liveness("test-agent")
+        
+        assert result.healthy is False
+        assert result.status == HealthStatus.UNHEALTHY
+    
+    @pytest.mark.asyncio
+    async def test_readiness_check_success(self, health_config):
+        """Test successful readiness check"""
+        monitor = HealthMonitor(config=health_config)
+        agent = MockAgent()
+        agent._is_ready = True
+        
+        monitor.register_agent("test-agent", agent)
+        result = await monitor._check_readiness("test-agent")
+        
+        assert result.healthy is True
+    
+    @pytest.mark.asyncio
+    async def test_readiness_check_failure(self, health_config):
+        """Test failed readiness check"""
+        monitor = HealthMonitor(config=health_config)
+        agent = MockAgent()
+        agent._is_ready = False
+        
+        monitor.register_agent("test-agent", agent)
+        result = await monitor._check_readiness("test-agent")
+        
+        assert result.healthy is False
+        assert result.status == HealthStatus.DEGRADED
+    
+    @pytest.mark.asyncio
+    async def test_custom_health_check(self, health_config):
+        """Test custom health check function"""
+        monitor = HealthMonitor(config=health_config)
+        
+        async def custom_liveness():
+            return True
+        
+        monitor.register_agent(
+            "test-agent",
+            MockAgent(),
+            custom_liveness=custom_liveness
+        )
+        
+        result = await monitor._check_liveness("test-agent")
+        assert result.healthy is True
+    
+    @pytest.mark.asyncio
+    async def test_unregister_agent(self, health_config):
+        """Test agent unregistration"""
+        monitor = HealthMonitor(config=health_config)
+        monitor.register_agent("test-agent", MockAgent())
+        
+        monitor.unregister_agent("test-agent")
+        
+        assert "test-agent" not in monitor._agents
+        assert monitor.get_agent_health("test-agent") == HealthStatus.UNKNOWN
+
+
+# ============================================================================
+# ACP-002: Auto-Recovery Tests
+# ============================================================================
+
+class TestAutoRecovery:
+    """Tests for auto-recovery functionality"""
+    
+    @pytest.mark.asyncio
+    async def test_register_agent(self, recovery_config):
+        """Test agent registration for recovery"""
+        recovery = AutoRecoveryManager(config=recovery_config)
+        
+        recovery.register_agent(
+            "test-agent",
+            factory=lambda: MockAgent()
+        )
+        
+        assert "test-agent" in recovery._agent_factories
+        assert recovery.get_restart_count("test-agent") == 0
+    
+    @pytest.mark.asyncio
+    async def test_handle_failure_recovery(self, recovery_config):
+        """Test successful recovery after failure"""
+        recovery = AutoRecoveryManager(config=recovery_config)
+        
+        recovery.register_agent(
+            "test-agent",
+            factory=lambda: MockAgent()
+        )
+        
+        new_agent = await recovery.handle_failure("test-agent", Exception("Test failure"))
+        
+        assert new_agent is not None
+        assert recovery.get_restart_count("test-agent") == 1
+    
+    @pytest.mark.asyncio
+    async def test_max_restarts_limit(self, recovery_config):
+        """Test max restarts limit is respected"""
+        recovery_config.max_restarts = 2
+        recovery = AutoRecoveryManager(config=recovery_config)
+        
+        fail_count = 0
+        def failing_factory():
+            nonlocal fail_count
+            fail_count += 1
+            if fail_count < 3:
+                raise Exception("Factory failure")
+            return MockAgent()
+        
+        recovery.register_agent("test-agent", factory=failing_factory)
+        
+        # Simulate multiple failures
+        recovery._restart_counts["test-agent"] = 2
+        
+        result = await recovery.handle_failure("test-agent")
+        
+        # Should not recover when at max restarts (depending on config)
+        # The behavior depends on on_max_restarts setting
+    
+    @pytest.mark.asyncio
+    async def test_recovery_history(self, recovery_config):
+        """Test recovery history tracking"""
+        recovery = AutoRecoveryManager(config=recovery_config)
+        recovery.register_agent("test-agent", factory=lambda: MockAgent())
+        
+        await recovery.handle_failure("test-agent")
+        
+        history = recovery.get_recovery_history("test-agent")
+        assert len(history) >= 1
+        assert any(e.event_type in ("failure", "recovery_success") for e in history)
+    
+    @pytest.mark.asyncio
+    async def test_reset_restart_count(self, recovery_config):
+        """Test manual restart count reset"""
+        recovery = AutoRecoveryManager(config=recovery_config)
+        recovery.register_agent("test-agent", factory=lambda: MockAgent())
+        
+        recovery._restart_counts["test-agent"] = 5
+        recovery.reset_restart_count("test-agent")
+        
+        assert recovery.get_restart_count("test-agent") == 0
+
+
+# ============================================================================
+# ACP-003: Circuit Breaker Tests
+# ============================================================================
+
+class TestCircuitBreaker:
+    """Tests for circuit breaker functionality"""
+    
+    @pytest.mark.asyncio
+    async def test_initial_state_closed(self):
+        """Test circuit breaker starts in closed state"""
+        breaker = CircuitBreaker(name="test")
+        assert breaker.state == CircuitState.CLOSED
+        assert breaker.is_closed is True
+    
+    @pytest.mark.asyncio
+    async def test_opens_after_failures(self):
+        """Test circuit opens after threshold failures"""
+        config = CircuitBreakerConfig(failure_threshold=3)
+        breaker = CircuitBreaker(name="test", config=config)
+        
+        for i in range(3):
+            await breaker._on_failure(Exception("Test failure"))
+        
+        assert breaker.state == CircuitState.OPEN
+        assert breaker.is_open is True
+    
+    @pytest.mark.asyncio
+    async def test_rejects_when_open(self):
+        """Test requests are rejected when circuit is open"""
+        breaker = CircuitBreaker(name="test", failure_threshold=1)
+        
+        await breaker._on_failure(Exception("Test failure"))
+        
+        with pytest.raises(CircuitBreakerOpenError):
+            await breaker._before_call()
+    
+    @pytest.mark.asyncio
+    async def test_half_open_after_timeout(self):
+        """Test circuit transitions to half-open after recovery timeout"""
+        config = CircuitBreakerConfig(
+            failure_threshold=1,
+            recovery_timeout_seconds=0.1
+        )
+        breaker = CircuitBreaker(name="test", config=config)
+        
+        await breaker._on_failure(Exception("Test failure"))
+        assert breaker.state == CircuitState.OPEN
+        
+        # Wait for recovery timeout
+        await asyncio.sleep(0.15)
+        
+        # Next call should transition to half-open
+        await breaker._before_call()
+        assert breaker.state == CircuitState.HALF_OPEN
+    
+    @pytest.mark.asyncio
+    async def test_closes_after_successful_half_open(self):
+        """Test circuit closes after successful calls in half-open"""
+        config = CircuitBreakerConfig(
+            failure_threshold=1,
+            success_threshold=2,
+            recovery_timeout_seconds=0.1
+        )
+        breaker = CircuitBreaker(name="test", config=config)
+        
+        # Open the circuit
+        await breaker._on_failure(Exception("Test failure"))
+        await asyncio.sleep(0.15)
+        
+        # Transition to half-open and succeed
+        await breaker._before_call()
+        await breaker._on_success()
+        await breaker._on_success()
+        
+        assert breaker.state == CircuitState.CLOSED
+    
+    @pytest.mark.asyncio
+    async def test_decorator_usage(self):
+        """Test circuit breaker as decorator"""
+        breaker = CircuitBreaker(name="test", failure_threshold=3)
+        
+        @breaker
+        async def test_func():
+            return "success"
+        
+        result = await test_func()
+        assert result == "success"
+    
+    @pytest.mark.asyncio
+    async def test_context_manager_usage(self):
+        """Test circuit breaker as context manager"""
+        breaker = CircuitBreaker(name="test")
+        
+        async with breaker:
+            pass  # Success
+        
+        metrics = breaker.get_metrics()
+        assert metrics.total_successes >= 1
+    
+    def test_manual_reset(self):
+        """Test manual circuit reset"""
+        breaker = CircuitBreaker(name="test")
+        breaker._state = CircuitState.OPEN
+        breaker._failure_count = 10
+        
+        breaker.reset()
+        
+        assert breaker.state == CircuitState.CLOSED
+        assert breaker._failure_count == 0
+    
+    def test_circuit_breaker_registry(self):
+        """Test circuit breaker registry"""
+        registry = CircuitBreakerRegistry()
+        
+        breaker1 = registry.get_or_create("agent-1")
+        breaker2 = registry.get_or_create("agent-1")
+        breaker3 = registry.get_or_create("agent-2")
+        
+        assert breaker1 is breaker2
+        assert breaker1 is not breaker3
+
+
+# ============================================================================
+# ACP-004: Agent Scaling Tests
+# ============================================================================
+
+class TestAgentScaler:
+    """Tests for agent scaling functionality"""
+    
+    @pytest.mark.asyncio
+    async def test_register_agent_type(self):
+        """Test agent type registration"""
+        scaler = AgentScaler()
+        
+        scaler.register_agent_type(
+            agent_type="test-agent",
+            factory=lambda: MockAgent(),
+            replicas=2
+        )
+        
+        assert "test-agent" in scaler._agent_types
+    
+    @pytest.mark.asyncio
+    async def test_scale_to(self):
+        """Test scaling to specific replica count"""
+        scaler = AgentScaler()
+        scaler.register_agent_type("test-agent", factory=lambda: MockAgent())
+        
+        await scaler.scale_to("test-agent", 3)
+        
+        assert scaler.get_replica_count("test-agent") == 3
+    
+    @pytest.mark.asyncio
+    async def test_scale_up(self):
+        """Test scaling up"""
+        scaler = AgentScaler()
+        scaler.register_agent_type("test-agent", factory=lambda: MockAgent())
+        
+        await scaler.scale_to("test-agent", 2)
+        await scaler.scale_up("test-agent", 2)
+        
+        assert scaler.get_replica_count("test-agent") == 4
+    
+    @pytest.mark.asyncio
+    async def test_scale_down(self):
+        """Test scaling down"""
+        scaler = AgentScaler()
+        scaler.register_agent_type("test-agent", factory=lambda: MockAgent())
+        
+        await scaler.scale_to("test-agent", 4)
+        await scaler.scale_down("test-agent", 2)
+        
+        assert scaler.get_replica_count("test-agent") == 2
+    
+    @pytest.mark.asyncio
+    async def test_get_replica_load_balanced(self):
+        """Test round-robin load balancing"""
+        scaler = AgentScaler()
+        scaler.register_agent_type("test-agent", factory=lambda: MockAgent())
+        
+        await scaler.scale_to("test-agent", 3)
+        
+        replicas = set()
+        for _ in range(6):
+            replica = await scaler.get_replica("test-agent")
+            replicas.add(id(replica))
+        
+        # Should have gotten all 3 replicas
+        assert len(replicas) == 3
+    
+    @pytest.mark.asyncio
+    async def test_max_replicas_limit(self):
+        """Test max replicas limit is respected"""
+        config = ScalingConfig(max_replicas=5)
+        scaler = AgentScaler()
+        scaler.register_agent_type("test-agent", factory=lambda: MockAgent(), config=config)
+        
+        await scaler.scale_to("test-agent", 10)
+        
+        assert scaler.get_replica_count("test-agent") == 5
+
+
+# ============================================================================
+# ACP-005: Distributed Coordination Tests
+# ============================================================================
+
+class TestDistributedCoordinator:
+    """Tests for distributed coordination functionality"""
+    
+    @pytest.mark.asyncio
+    async def test_single_node_becomes_leader(self):
+        """Test single node becomes leader"""
+        coordinator = DistributedCoordinator(node_id="node-1", peers=[])
+        
+        await coordinator.start()
+        await asyncio.sleep(0.1)  # Allow election to complete
+        
+        assert coordinator.is_leader is True
+        assert coordinator.role == CoordinationRole.LEADER
+        
+        await coordinator.stop()
+    
+    @pytest.mark.asyncio
+    async def test_acquire_release_lock(self):
+        """Test distributed lock acquisition and release"""
+        coordinator = DistributedCoordinator(node_id="node-1")
+        
+        acquired = await coordinator.acquire_lock("resource-1", timeout=1.0)
+        assert acquired is True
+        
+        coordinator.release_lock("resource-1")
+        
+        # Should be able to acquire again
+        acquired = await coordinator.acquire_lock("resource-1", timeout=1.0)
+        assert acquired is True
+    
+    @pytest.mark.asyncio
+    async def test_lock_context_manager(self):
+        """Test lock as context manager"""
+        coordinator = DistributedCoordinator(node_id="node-1")
+        
+        async with coordinator.lock("resource-1"):
+            # Lock should be held
+            assert "resource-1" in coordinator._lock_holders
+        
+        # Lock should be released
+        assert "resource-1" not in coordinator._lock_holders
+    
+    @pytest.mark.asyncio
+    async def test_get_leader_info(self):
+        """Test getting leader information"""
+        coordinator = DistributedCoordinator(node_id="node-1", peers=[])
+        
+        await coordinator.start()
+        await asyncio.sleep(0.1)
+        
+        info = coordinator.get_leader_info()
+        
+        assert info is not None
+        assert info.leader_id == "node-1"
+        
+        await coordinator.stop()
+
+
+# ============================================================================
+# ACP-006: Dependency Graph Tests
+# ============================================================================
+
+class TestDependencyGraph:
+    """Tests for dependency graph functionality"""
+    
+    def test_add_agent(self):
+        """Test adding agents to the graph"""
+        graph = DependencyGraph()
+        
+        graph.add_agent("api-server", depends_on=["database", "cache"])
+        graph.add_agent("database", depends_on=[])
+        graph.add_agent("cache", depends_on=[])
+        
+        assert len(graph._agents) == 3
+    
+    def test_get_dependencies(self):
+        """Test getting agent dependencies"""
+        graph = DependencyGraph()
+        graph.add_agent("api-server", depends_on=["database", "cache"])
+        
+        deps = graph.get_dependencies("api-server")
+        
+        assert "database" in deps
+        assert "cache" in deps
+    
+    def test_get_dependents(self):
+        """Test getting agents that depend on an agent"""
+        graph = DependencyGraph()
+        graph.add_agent("database", depends_on=[])
+        graph.add_agent("api-server", depends_on=["database"])
+        graph.add_agent("worker", depends_on=["database"])
+        
+        dependents = graph.get_dependents("database")
+        
+        assert "api-server" in dependents
+        assert "worker" in dependents
+    
+    def test_circular_dependency_detection(self):
+        """Test circular dependency detection"""
+        graph = DependencyGraph()
+        graph.add_agent("a", depends_on=["b"])
+        graph.add_agent("b", depends_on=["c"])
+        graph.add_agent("c", depends_on=["a"])  # Circular!
+        
+        assert graph.has_circular_dependency() is True
+    
+    def test_no_circular_dependency(self):
+        """Test no circular dependency in valid graph"""
+        graph = DependencyGraph()
+        graph.add_agent("a", depends_on=["b"])
+        graph.add_agent("b", depends_on=["c"])
+        graph.add_agent("c", depends_on=[])
+        
+        assert graph.has_circular_dependency() is False
+    
+    def test_startup_order(self):
+        """Test correct startup order"""
+        graph = DependencyGraph()
+        graph.add_agent("api-server", depends_on=["database", "cache"])
+        graph.add_agent("database", depends_on=[])
+        graph.add_agent("cache", depends_on=[])
+        
+        order = graph.get_startup_order()
+        
+        # database and cache should come before api-server
+        assert order.index("database") < order.index("api-server")
+        assert order.index("cache") < order.index("api-server")
+    
+    def test_parallel_startup_groups(self):
+        """Test parallel startup group generation"""
+        graph = DependencyGraph()
+        graph.add_agent("api-server", depends_on=["database", "cache"])
+        graph.add_agent("database", depends_on=[])
+        graph.add_agent("cache", depends_on=[])
+        
+        groups = graph.get_parallel_startup_groups()
+        
+        assert len(groups) == 2
+        # First group should contain database and cache
+        assert set(groups[0]) == {"database", "cache"}
+        # Second group should contain api-server
+        assert groups[1] == ["api-server"]
+    
+    def test_shutdown_order(self):
+        """Test shutdown order is reverse of startup"""
+        graph = DependencyGraph()
+        graph.add_agent("api-server", depends_on=["database"])
+        graph.add_agent("database", depends_on=[])
+        
+        startup = graph.get_startup_order()
+        shutdown = graph.get_shutdown_order()
+        
+        assert shutdown == list(reversed(startup))
+    
+    def test_validate_missing_dependency(self):
+        """Test validation catches missing dependencies"""
+        graph = DependencyGraph()
+        graph.add_agent("api-server", depends_on=["database"])
+        # database not added
+        
+        errors = graph.validate()
+        
+        assert len(errors) > 0
+        assert any("missing" in e.lower() for e in errors)
+
+
+# ============================================================================
+# ACP-007: Graceful Shutdown Tests
+# ============================================================================
+
+class TestGracefulShutdown:
+    """Tests for graceful shutdown functionality"""
+    
+    @pytest.mark.asyncio
+    async def test_register_operation(self):
+        """Test registering in-flight operations"""
+        manager = GracefulShutdownManager()
+        
+        op_id = manager.register_operation(
+            agent_id="test-agent",
+            operation_type="verification",
+            data={"claim_id": "123"}
+        )
+        
+        assert op_id is not None
+        assert manager.get_in_flight_count() == 1
+    
+    @pytest.mark.asyncio
+    async def test_complete_operation(self):
+        """Test completing operations"""
+        manager = GracefulShutdownManager()
+        
+        op_id = manager.register_operation("test-agent", "verification")
+        manager.complete_operation(op_id)
+        
+        assert manager.get_in_flight_count() == 0
+    
+    @pytest.mark.asyncio
+    async def test_shutdown_drains_operations(self):
+        """Test shutdown waits for operations to complete"""
+        config = ShutdownConfig(drain_timeout_seconds=1.0)
+        manager = GracefulShutdownManager(config=config)
+        
+        op_id = manager.register_operation("test-agent", "verification")
+        
+        # Complete operation after short delay
+        async def complete_later():
+            await asyncio.sleep(0.1)
+            manager.complete_operation(op_id)
+        
+        asyncio.create_task(complete_later())
+        
+        result = await manager.shutdown()
+        
+        assert result["status"] == "stopped" or manager.phase == ShutdownPhase.TERMINATED
+    
+    @pytest.mark.asyncio
+    async def test_shutdown_hooks(self):
+        """Test shutdown hooks are called"""
+        manager = GracefulShutdownManager()
+        
+        hook_called = False
+        
+        async def shutdown_hook():
+            nonlocal hook_called
+            hook_called = True
+        
+        manager.add_shutdown_hook(shutdown_hook)
+        await manager.shutdown()
+        
+        assert hook_called is True
+    
+    @pytest.mark.asyncio
+    async def test_no_new_operations_during_shutdown(self):
+        """Test new operations are rejected during shutdown"""
+        manager = GracefulShutdownManager()
+        manager._phase = ShutdownPhase.DRAINING
+        
+        with pytest.raises(RuntimeError):
+            manager.register_operation("test-agent", "verification")
+
+
+# ============================================================================
+# ACP-008: Resource Quota Tests
+# ============================================================================
+
+class TestResourceQuotas:
+    """Tests for resource quota functionality"""
+    
+    def test_set_quota(self):
+        """Test setting resource quota"""
+        manager = ResourceQuotaManager()
+        
+        quota = AgentResourceQuota(
+            memory_mb=512,
+            cpu_percent=25.0,
+            max_concurrent_operations=10
+        )
+        
+        manager.set_quota("test-agent", quota)
+        
+        assert manager.get_quota("test-agent") is not None
+    
+    def test_can_execute_within_quota(self):
+        """Test execution allowed within quota"""
+        manager = ResourceQuotaManager()
+        manager.set_quota("test-agent", AgentResourceQuota(
+            max_concurrent_operations=10
+        ))
+        
+        assert manager.can_execute("test-agent") is True
+    
+    def test_cannot_execute_at_limit(self):
+        """Test execution denied at concurrent limit"""
+        manager = ResourceQuotaManager()
+        manager.set_quota("test-agent", AgentResourceQuota(
+            max_concurrent_operations=2
+        ))
+        
+        # Record 2 operations
+        manager.record_operation_start("test-agent")
+        manager.record_operation_start("test-agent")
+        
+        assert manager.can_execute("test-agent") is False
+    
+    def test_operation_tracking(self):
+        """Test operation start/end tracking"""
+        manager = ResourceQuotaManager()
+        manager.set_quota("test-agent", AgentResourceQuota())
+        
+        manager.record_operation_start("test-agent")
+        usage = manager.get_usage("test-agent")
+        assert usage.concurrent_operations == 1
+        
+        manager.record_operation_end("test-agent")
+        usage = manager.get_usage("test-agent")
+        assert usage.concurrent_operations == 0
+    
+    def test_resource_usage_update(self):
+        """Test resource usage updates"""
+        manager = ResourceQuotaManager()
+        manager.set_quota("test-agent", AgentResourceQuota())
+        
+        manager.update_resource_usage("test-agent", memory_mb=256.0, cpu_percent=15.0)
+        
+        usage = manager.get_usage("test-agent")
+        assert usage.memory_mb == 256.0
+        assert usage.cpu_percent == 15.0
+    
+    def test_check_quota_violations(self):
+        """Test quota violation detection"""
+        manager = ResourceQuotaManager()
+        manager.set_quota("test-agent", AgentResourceQuota(
+            memory_mb=512,
+            cpu_percent=25.0
+        ))
+        
+        # Exceed memory
+        manager.update_resource_usage("test-agent", memory_mb=600.0)
+        
+        violations = manager.check_quota_violations()
+        
+        assert "test-agent" in violations
+        assert any("Memory" in v for v in violations["test-agent"])
+
+
+# ============================================================================
+# ACP-009: Observability Tests
+# ============================================================================
+
+class TestAgentObservability:
+    """Tests for agent observability functionality"""
+    
+    def test_record_metric(self):
+        """Test metric recording"""
+        provider = AgentObservabilityProvider()
+        
+        provider.record_metric(
+            agent_id="test-agent",
+            name="latency_ms",
+            value=150.5,
+            labels={"operation": "verify"}
+        )
+        
+        metrics = provider.get_metrics("test-agent")
+        assert len(metrics) > 0
+        assert metrics[0].name == "latency_ms"
+    
+    def test_increment_counter(self):
+        """Test counter increment"""
+        provider = AgentObservabilityProvider()
+        
+        provider.increment_counter("test-agent", "requests_total")
+        provider.increment_counter("test-agent", "requests_total")
+        
+        # Counter should have value 2
+        metrics = provider.get_metrics("test-agent", "requests_total")
+        assert len(metrics) == 2
+    
+    def test_set_gauge(self):
+        """Test gauge setting"""
+        provider = AgentObservabilityProvider()
+        
+        provider.set_gauge("test-agent", "active_connections", 5.0)
+        
+        metrics = provider.get_metrics("test-agent", "active_connections")
+        assert len(metrics) == 1
+        assert metrics[0].value == 5.0
+    
+    def test_log_entry(self):
+        """Test log entry creation"""
+        provider = AgentObservabilityProvider()
+        
+        provider.log(
+            agent_id="test-agent",
+            level="info",
+            message="Test message",
+            context={"key": "value"}
+        )
+        
+        logs = provider.get_logs("test-agent")
+        assert len(logs) == 1
+        assert logs[0].message == "Test message"
+    
+    def test_prometheus_export(self):
+        """Test Prometheus format export"""
+        provider = AgentObservabilityProvider()
+        
+        provider.increment_counter("test-agent", "requests")
+        provider.set_gauge("test-agent", "memory_usage", 256.0)
+        
+        prometheus_text = provider.export_prometheus()
+        
+        assert "requests" in prometheus_text
+        assert "memory_usage" in prometheus_text
+    
+    def test_agent_summary(self):
+        """Test agent summary generation"""
+        provider = AgentObservabilityProvider()
+        
+        provider.record_metric("test-agent", "metric1", 1.0)
+        provider.log("test-agent", "info", "Message 1")
+        
+        summary = provider.get_agent_summary("test-agent")
+        
+        assert summary["agent_id"] == "test-agent"
+        assert summary["total_metrics"] > 0
+        assert summary["total_logs"] > 0
+
+
+# ============================================================================
+# ACP-010: Hot Reload Tests
+# ============================================================================
+
+class TestHotReload:
+    """Tests for hot reload functionality"""
+    
+    def test_register_agent(self):
+        """Test agent registration for hot reload"""
+        manager = HotReloadManager()
+        
+        manager.register_agent(
+            agent_id="test-agent",
+            module_name="test_module",
+            class_name="TestAgent"
+        )
+        
+        assert "test-agent" in manager._agents
+    
+    @pytest.mark.asyncio
+    async def test_check_for_changes_no_change(self):
+        """Test change detection when no changes"""
+        manager = HotReloadManager()
+        manager.register_agent("test-agent", "test_module", "TestAgent")
+        
+        # Force a version
+        manager._versions["test-agent"] = "abc123"
+        
+        # Should detect change since computed version will differ
+        # (module doesn't exist, so version will be different)
+        has_changes = await manager.check_for_changes("test-agent")
+        # Behavior depends on whether module exists
+    
+    def test_get_agent_version(self):
+        """Test getting agent version"""
+        manager = HotReloadManager()
+        manager.register_agent("test-agent", "test_module", "TestAgent")
+        
+        version = manager.get_agent_version("test-agent")
+        
+        assert version is not None
+    
+    def test_reload_history(self):
+        """Test reload history tracking"""
+        manager = HotReloadManager()
+        
+        history = manager.get_reload_history()
+        
+        assert isinstance(history, list)
+
+
+# ============================================================================
+# Enhanced Control Plane Tests
+# ============================================================================
+
+class TestEnhancedControlPlane:
+    """Tests for the enhanced agent control plane"""
+    
+    def test_create_control_plane(self):
+        """Test control plane creation"""
+        control_plane = create_control_plane(
+            health_check_interval=30.0,
+            auto_recovery=True
+        )
+        
+        assert control_plane is not None
+        assert isinstance(control_plane, EnhancedAgentControlPlane)
+    
+    def test_register_agent(self):
+        """Test agent registration"""
+        control_plane = create_control_plane()
+        
+        agent_id = control_plane.register(
+            MockAgent,
+            replicas=2,
+            dependencies=[]
+        )
+        
+        assert agent_id == "MockAgent"
+        assert "MockAgent" in control_plane._registrations
+    
+    def test_register_with_resources(self):
+        """Test agent registration with resource quota"""
+        control_plane = create_control_plane()
+        
+        control_plane.register(
+            MockAgent,
+            replicas=1,
+            resources=AgentResourceQuota(
+                memory_mb=512,
+                cpu_percent=25
+            )
+        )
+        
+        quota = control_plane.quota_manager.get_quota("MockAgent")
+        assert quota is not None
+        assert quota.memory_mb == 512
+    
+    def test_register_with_circuit_breaker(self):
+        """Test agent registration with circuit breaker"""
+        control_plane = create_control_plane()
+        
+        breaker = CircuitBreaker(
+            name="test",
+            failure_threshold=5,
+            recovery_timeout=60
+        )
+        
+        control_plane.register(
+            MockAgent,
+            circuit_breaker=breaker
+        )
+        
+        registered_breaker = control_plane.get_circuit_breaker("MockAgent")
+        assert registered_breaker is breaker
+    
+    @pytest.mark.asyncio
+    async def test_start_all(self):
+        """Test starting all agents"""
+        control_plane = create_control_plane()
+        
+        control_plane.register(MockAgent, replicas=1)
+        
+        result = await control_plane.start_all()
+        
+        assert result["status"] == "started"
+        
+        await control_plane.stop_all()
+    
+    @pytest.mark.asyncio
+    async def test_stop_all(self):
+        """Test stopping all agents"""
+        control_plane = create_control_plane()
+        control_plane.register(MockAgent, replicas=1)
+        
+        await control_plane.start_all()
+        result = await control_plane.stop_all()
+        
+        assert result["status"] == "stopped"
+    
+    def test_get_status(self):
+        """Test getting control plane status"""
+        control_plane = create_control_plane()
+        control_plane.register(MockAgent)
+        
+        status = control_plane.get_status()
+        
+        assert "running" in status
+        assert "registered_agents" in status
+        assert "MockAgent" in status["registered_agents"]
+
+
+# ============================================================================
+# Integration Tests
+# ============================================================================
+
+class TestIntegration:
+    """Integration tests for lifecycle management"""
+    
+    @pytest.mark.asyncio
+    async def test_full_lifecycle(self):
+        """Test complete agent lifecycle"""
+        control_plane = create_control_plane(
+            health_check_interval=1.0,
+            auto_recovery=True
+        )
+        
+        # Register agents with dependencies
+        control_plane.register(MockAgent, agent_id="database", replicas=1)
+        control_plane.register(
+            MockAgent,
+            agent_id="api-server",
+            replicas=2,
+            dependencies=["database"],
+            resources=AgentResourceQuota(memory_mb=256)
+        )
+        
+        # Start all
+        result = await control_plane.start_all()
+        assert result["status"] == "started"
+        
+        # Check health
+        health = control_plane.get_all_health_status()
+        assert len(health) > 0
+        
+        # Get metrics
+        metrics = control_plane.get_metrics()
+        assert isinstance(metrics, str)
+        
+        # Stop all
+        result = await control_plane.stop_all()
+        assert result["status"] == "stopped"
+    
+    @pytest.mark.asyncio
+    async def test_dependency_order(self):
+        """Test agents start in dependency order"""
+        control_plane = create_control_plane()
+        
+        start_order = []
+        
+        class TrackedAgent(MockAgent):
+            async def start(self):
+                start_order.append(self.name)
+                await super().start()
+        
+        control_plane.register(
+            lambda: TrackedAgent("cache"),
+            agent_id="cache",
+            replicas=1
+        )
+        control_plane.register(
+            lambda: TrackedAgent("database"),
+            agent_id="database",
+            replicas=1
+        )
+        control_plane.register(
+            lambda: TrackedAgent("api"),
+            agent_id="api",
+            replicas=1,
+            dependencies=["database", "cache"]
+        )
+        
+        # The dependency graph should ensure database and cache start before api
+        # (Note: this is validated by the dependency graph tests above)
+
+
+if __name__ == "__main__":
+    pytest.main([__file__, "-v"])