hud-python 0.4.45__py3-none-any.whl → 0.5.13__py3-none-any.whl

hud/otel/tests/test_processors.py

@@ -1,197 +0,0 @@
-"""Tests for OpenTelemetry processors."""
-
-from __future__ import annotations
-
-from unittest.mock import MagicMock, patch
-
-from hud.otel.processors import HudEnrichmentProcessor
-
-
-class TestHudEnrichmentProcessor:
-    """Test HudEnrichmentProcessor."""
-
-    def test_on_start_with_run_id(self):
-        """Test on_start with current task run ID."""
-
-        processor = HudEnrichmentProcessor()
-
-        # Mock span
-        span = MagicMock()
-        span.set_attribute = MagicMock()
-        span.is_recording.return_value = True
-
-        # Mock baggage to return run ID
-        parent_context = {}
-        with patch("hud.otel.processors.baggage.get_baggage") as mock_get_baggage:
-            # Return run ID for task_run_id, None for job_id
-            mock_get_baggage.side_effect = (
-                lambda key, context: "test-run-123" if key == "hud.task_run_id" else None
-            )
-            processor.on_start(span, parent_context)
-
-        # Verify attribute was set
-        span.set_attribute.assert_called_with("hud.task_run_id", "test-run-123")
-
-    def test_on_start_no_run_id(self):
-        """Test on_start without current task run ID."""
-
-        processor = HudEnrichmentProcessor()
-
-        # Mock span
-        span = MagicMock()
-        span.set_attribute = MagicMock()
-        span.is_recording.return_value = True
-        span.name = "test_span"
-
-        # Set up attributes to return None (not matching any step type)
-        span.attributes = {}
-
-        # Mock baggage to return None
-        parent_context = {}
-        with patch("hud.otel.processors.baggage.get_baggage", return_value=None):
-            processor.on_start(span, parent_context)
-
-        # Verify only step count attributes were set (no run_id or job_id)
-        calls = span.set_attribute.call_args_list
-        set_attrs = {call[0][0] for call in calls}
-
-        # Should have step counts but not run_id/job_id
-        assert "hud.task_run_id" not in set_attrs
-        assert "hud.job_id" not in set_attrs
-        assert "hud.base_mcp_steps" in set_attrs
-        assert "hud.mcp_tool_steps" in set_attrs
-        assert "hud.agent_steps" in set_attrs
-
-    def test_on_end(self):
-        """Test on_end does nothing."""
-
-        processor = HudEnrichmentProcessor()
-        span = MagicMock()
-
-        # Should not raise
-        processor.on_end(span)
-
-    def test_shutdown(self):
-        """Test shutdown does nothing."""
-
-        processor = HudEnrichmentProcessor()
-
-        # Should not raise
-        processor.shutdown()
-
-    def test_force_flush(self):
-        """Test force_flush returns True."""
-
-        processor = HudEnrichmentProcessor()
-
-        # Should return True
-        result = processor.force_flush()
-        assert result is True
-
-    def test_on_start_with_job_id(self):
-        """Test on_start with job ID in baggage."""
-
-        processor = HudEnrichmentProcessor()
-
-        # Mock span
-        span = MagicMock()
-        span.set_attribute = MagicMock()
-        span.is_recording.return_value = True
-
-        # Mock baggage with job ID
-        parent_context = {}
-        with patch("hud.otel.processors.baggage.get_baggage") as mock_get_baggage:
-            # Return None for task_run_id, job-123 for job_id
-            mock_get_baggage.side_effect = (
-                lambda key, context: "job-123" if key == "hud.job_id" else None
-            )
-            processor.on_start(span, parent_context)
-
-        # Verify job ID attribute was set
-        span.set_attribute.assert_called_with("hud.job_id", "job-123")
-
-    def test_on_start_exception_handling(self):
-        """Test on_start handles exceptions gracefully."""
-
-        processor = HudEnrichmentProcessor()
-
-        # Mock span that raises exception
-        span = MagicMock()
-        span.is_recording.side_effect = Exception("Test error")
-
-        # Should not raise
-        processor.on_start(span, parent_context=None)
-
-    def test_on_start_exception_handling_extended(self):
-        """Test that exceptions in on_start are caught and logged."""
-        from hud.otel.processors import HudEnrichmentProcessor
-
-        processor = HudEnrichmentProcessor()
-
-        # Create a mock span that raises when setting attributes
-        mock_span = MagicMock()
-        mock_span.is_recording.return_value = True
-        mock_span.set_attribute.side_effect = RuntimeError("Attribute error")
-
-        parent_context = {}
-
-        # Patch logger and baggage to force an exception when setting attribute
-        with (
-            patch("hud.otel.processors.logger") as mock_logger,
-            patch("hud.otel.processors.baggage.get_baggage", return_value="test-id"),
-        ):
-            # Should not raise, exception should be caught
-            processor.on_start(mock_span, parent_context)
-
-            # Verify logger.debug was called with the exception
-            mock_logger.debug.assert_called_once()
-            args = mock_logger.debug.call_args[0]
-            assert "HudEnrichmentProcessor.on_start error" in args[0]
-            assert "Attribute error" in str(args[1])
-
-    def test_on_start_with_baggage_get_exception(self):
-        """Test exception handling when baggage.get_baggage fails for task_run_id."""
-        processor = HudEnrichmentProcessor()
-
-        mock_span = MagicMock()
-        mock_span.is_recording.return_value = True
-
-        parent_context = {}
-
-        # Make baggage.get_baggage raise an exception for task_run_id
-        with (
-            patch(
-                "hud.otel.processors.baggage.get_baggage",
-                side_effect=ValueError("Context error"),
-            ),
-            patch("hud.otel.processors.logger") as mock_logger,
-        ):
-            # Should not raise
-            processor.on_start(mock_span, parent_context)
-
-            # Verify logger.debug was called
-            mock_logger.debug.assert_called_once()
-            args = mock_logger.debug.call_args[0]
-            assert "Context error" in str(args[1])
-
-    def test_on_start_with_baggage_exception(self):
-        """Test exception handling when baggage.get_baggage fails."""
-        processor = HudEnrichmentProcessor()
-
-        mock_span = MagicMock()
-        mock_span.is_recording.return_value = True
-
-        parent_context = {}
-
-        # Make baggage.get_baggage raise an exception
-        with (
-            patch("hud.otel.processors.baggage.get_baggage", side_effect=KeyError("Baggage error")),
-            patch("hud.otel.processors.logger") as mock_logger,
-        ):
-            # Should not raise
-            processor.on_start(mock_span, parent_context)
-
-            # Verify logger.debug was called
-            mock_logger.debug.assert_called_once()
-            args = mock_logger.debug.call_args[0]
-            assert "Baggage error" in str(args[1])

hud/rl/README.md

@@ -1,30 +0,0 @@
-We suggest running hud rl (or with the --local flag) for optimal hyperparameters and native HuggingFace running.
-
-However, to run this independently, sping up an instance with at least 2 GPUs and run:
-```bash
-sudo apt-get update -y && sudo apt-get install -y cuda-toolkit-12-6
-uv pip install -e .[rl]
-uv pip install ninja
-uv pip install flash-attn --no-build-isolation
-```
-
-Launch a vllm server with:
-```bash
-export VLLM_ALLOW_RUNTIME_LORA_UPDATING=True
-export TOKENIZERS_PARALLELISM=false
-export VLLM_LOGGING_LEVEL=INFO
-export CUDA_VISIBLE_DEVICES=7 # Set this to your last GPU
-
-uv run vllm serve Qwen/Qwen2.5-VL-3B-Instruct \
-    --api-key token-abc123 --host 0.0.0.0 --port 8000 --tensor-parallel-size 1 --trust-remote-code \
-    --max-model-len 16384 --enable-lora --max-lora-rank 64 --max-cpu-loras 4 --enable-auto-tool-choice \
-    --tool-call-parser hermes --disable-log-requests --dtype auto
-```
-
-And training with (replace 2 with your spare GPUs):
-```bash
-hud get hud-evals/2048-basic
-torchrun --nproc-per-node 2 -m hud.rl.train --tasks 2048-basic.json --verbose
-```
-
-Add a `--config path/to/config.json` flag to run a specific configuration (or change the defaults in config.py)

hud/rl/__init__.py

@@ -1 +0,0 @@
-"""RL module for HUD."""

hud/rl/actor.py

@@ -1,176 +0,0 @@
-"""Actor for episode collection using vLLM and HUD."""
-
-from __future__ import annotations
-
-import asyncio
-import logging
-
-import httpx
-from openai import AsyncOpenAI
-
-import hud
-from hud.agents.openai_chat_generic import GenericOpenAIChatAgent
-from hud.clients.utils.retry_transport import create_retry_httpx_client
-from hud.types import Task, Trace
-from hud.utils.hud_console import HUDConsole
-
-from .config import Config
-
-logger = logging.getLogger(__name__)
-hud_console = HUDConsole(logger)
-
-
-class Actor:
-    """Collects episodes using vLLM-served models via HUD agents."""
-
-    def __init__(self, config: Config) -> None:
-        self.config = config
-        self.actor_config = config.actor
-        self.current_adapter = config.model.base_model
-
-        # Setup OpenAI client for vLLM
-        base_url = self.actor_config.vllm_base_url.replace("localhost", "127.0.0.1")
-        self.openai_client = self._create_openai_client(base_url)
-
-    def _create_openai_client(self, base_url: str) -> AsyncOpenAI:
-        """Create OpenAI client with optimized settings for vLLM."""
-        # Match connection limits to parallel_episodes to avoid bottlenecks
-        # Use shorter per-request timeout and keep retries modest to avoid long blocking
-        http_client = create_retry_httpx_client(
-            timeout=httpx.Timeout(30.0),
-        )
-        return AsyncOpenAI(
-            base_url=base_url,
-            api_key=self.actor_config.vllm_api_key,
-            http_client=http_client,
-            max_retries=2,
-        )
-
-    def create_agent(self) -> GenericOpenAIChatAgent:
-        """Create an agent with the current adapter."""
-        return GenericOpenAIChatAgent(
-            openai_client=self.openai_client,
-            model_name=self.current_adapter,
-            allowed_tools=self.actor_config.allowed_tools,
-            append_setup_output=False,
-            system_prompt=self.actor_config.system_prompt,
-            verbose=self.config.verbose,
-            completion_kwargs={
-                "temperature": self.actor_config.temperature,
-                "max_tokens": self.actor_config.max_new_tokens,
-                "tool_choice": "required" if self.actor_config.force_tool_choice else "auto",
-            },
-        )
-
-    def update_adapter(self, adapter_name: str) -> None:
-        """Update the current adapter being used."""
-        self.current_adapter = adapter_name
-        hud_console.info(f"[Actor] Using adapter: {adapter_name}")
-
-    async def run_tasks(self, tasks: list[Task], job_id: str) -> list[Trace]:
-        """Run tasks and collect traces."""
-        traces = []
-
-        # Process tasks in batches respecting max_parallel_episodes limit
-        for batch_start in range(0, len(tasks), self.actor_config.max_parallel_episodes):
-            batch_end = min(batch_start + self.actor_config.max_parallel_episodes, len(tasks))
-            batch = tasks[batch_start:batch_end]
-
-            # Run batch in parallel with per-episode timeout protection
-            async def run_with_timeout(t: Task) -> Trace:
-                try:
-                    return await asyncio.wait_for(
-                        self._run_task(t, job_id),
-                        timeout=self.actor_config.episode_timeout_sec,
-                    )
-                except TimeoutError:
-                    hud_console.warning_log(f"Episode timed out for task {t.id}")
-                    # Attach task so buffer grouping has key
-                    return Trace(isError=True, content="Episode timeout", task=t)
-
-            results = await asyncio.gather(
-                *[run_with_timeout(t) for t in batch],
-                return_exceptions=True,
-            )
-
-            # Normalize exceptions to error traces and ensure task is attached
-            for t, res in zip(batch, results, strict=False):
-                if isinstance(res, Exception):
-                    hud_console.warning_log(f"Episode error: {res}")
-                    traces.append(Trace(isError=True, content=str(res), task=t))
-                else:
-                    traces.append(res)
-
-        return traces
-
-    async def _run_task(self, task: Task, job_id: str) -> Trace:
-        """Run a single task."""
-        agent = self.create_agent()
-
-        # Run the task
-        try:
-            with hud.trace(f"Training | {task.prompt}", job_id=job_id):
-                result = await agent.run(task, max_steps=self.actor_config.max_steps_per_episode)
-
-        except Exception:
-            logger.info("GOT EXCEPTION")
-            # Preserve task on exception for grouping
-            return Trace(isError=True, task=task)
-
-        result.info["tool_spec"] = agent.get_tool_schemas()
-
-        return result
-
-
-if __name__ == "__main__":
-    from hud.types import Task
-
-    async def test_actor() -> None:
-        """Test the actor with a single 2048 task using local hud-browser image."""
-        config = Config()
-        config.actor.max_parallel_episodes = 1
-        config.actor.max_steps_per_episode = 6
-        config.actor.episode_timeout_sec = 120
-        config.verbose = True
-
-        # Create test task with local hud-browser image
-        task_data = {
-            "id": "test_2048_128",
-            "prompt": "Play the browser-based 2048 game and try to reach the 128 tile. Start by taking a screenshot, then make strategic moves using arrow keys.",  # noqa: E501
-            "mcp_config": {
-                "local": {
-                    "command": "sh",
-                    "args": [
-                        "-c",
-                        "docker run --rm --platform linux/amd64 -i hud-browser:latest 2>/dev/null",
-                    ],
-                }
-            },
-            "setup_tool": {"name": "launch_app", "arguments": {"app_name": "2048"}},
-            "evaluate_tool": {
-                "name": "evaluate",
-                "arguments": {"name": "game_2048_max_number", "arguments": {"target": 128}},
-            },
-            "system_prompt": "You are an expert 2048 game player. Use arrow keys to reach the target tile. First take a screenshot, then make strategic moves.",  # noqa: E501
-        }
-
-        task = Task(**task_data)
-        actor = Actor(config)
-
-        logger.info("Testing actor with task: %s", task.id)
-        logger.info("Model: %s", config.model.base_model)
-        logger.info("VLLM: %s", config.actor.vllm_base_url)
-
-        traces = await actor.run_tasks([task], job_id="test_2048")
-
-        for trace in traces:
-            if trace.isError:
-                logger.info("Error: %s", trace.content)
-            else:
-                logger.info("Success!")
-                logger.info("Trace info: %s", trace.info if hasattr(trace, "info") else "No info")
-                # Check for evaluation in the trace info
-                if hasattr(trace, "info") and "evaluation" in trace.info:
-                    logger.info("  Evaluation: %s", trace.info["evaluation"])
-
-    asyncio.run(test_actor())