nooa-bench 0.0.7__py3-none-any.whl

nooa_bench/__init__.py

@@ -0,0 +1,19 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+"""Benchmark agent and Harbor runner for the NOOA framework.
+
+Minimal reproducibility package for the NOOA tech report: the benchmark-agnostic
+``BenchAgent`` (SWE-bench Verified, Terminal-Bench 2.0), the ``nemo-harbor``
+container runner, and the trace analyzer used to extract the per-task token
+statistics reported in the paper.
+"""
+
+from __future__ import annotations
+
+# Maps --agent-type CLI values to dotted import paths: "module:ClassName"
+AGENT_CLASSES: dict[str, str] = {
+    # Unified SWE-bench + Terminal-Bench agent (the tech report's BenchAgent)
+    "bench": "nooa_bench.bench_agent:BenchAgent",
+}
+
+__all__ = ["AGENT_CLASSES"]

nooa_bench/bench_agent.py

@@ -0,0 +1,256 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+"""
+Generic benchmark agent for code and system tasks.
+
+A single, non-specialized CodeAct agent that works on any task requiring shell
+access inside a container. Not tuned for any particular benchmark -- the same
+agent handles SWE-bench, Terminal-Bench, or any Harbor-compatible task.
+
+Core contract:
+- ``self.shell`` for persistent shell access (run/read/replace/write_file)
+- ``self.repo`` for code navigation that returns ShellTools Match anchors
+- ``self.todo`` for optional structured progress tracking
+- Structured return: the agent must declare solution_description, evidence,
+  and command_to_verify when finishing -- forcing reflection before return.
+"""
+
+from __future__ import annotations
+
+from nooa import hidden as _hidden
+
+_agentdoc_hidden_names = {"_hidden"}
+
+with _hidden:
+    import logging
+    import os
+    from typing import TYPE_CHECKING, Any
+
+    from nooa_cli.tools.repo_tools import RepoTools
+    from pydantic import BaseModel, Field
+
+    from nooa import Agent, CodeActStrategy, strategy
+    from nooa.agentdoc import doc, spec
+    from nooa.config import CodeActConfig
+    from nooa.context_blocks import DynamicContext
+    from nooa.tools.shell_tools import ShellTools
+    from nooa.tools.todo import TodoManager
+    from nooa.unifiedllm import FakeLLMClient
+
+if TYPE_CHECKING:
+    from nooa.unifiedllm import UnifiedLLM
+
+_logger = logging.getLogger(__name__)
+
+_OPTIONAL_TESTBED_ACTIVATE = (
+    "if [ -f /opt/miniconda3/etc/profile.d/conda.sh ]; then "
+    "[ -d /opt/harbor/cpython312/bin ] && export PATH=/opt/harbor/cpython312/bin:$PATH; "
+    "source /opt/miniconda3/etc/profile.d/conda.sh; "
+    "conda env list | awk '{print $1}' | grep -qx testbed && conda activate testbed || true; "
+    "fi"
+)
+
+
+class TaskResult(BaseModel):
+    """Structured result the agent must return when finishing a task."""
+
+    solution_description: str = Field(
+        description="What you did and why it solves the problem. Describe root cause and fix."
+    )
+    evidence: str = Field(
+        description=(
+            "Concrete evidence that the task is done: what tests passed, "
+            "what output was produced, what behavior changed. Not a guess -- "
+            "cite the actual shell output you observed."
+        )
+    )
+    command_to_verify: str = Field(
+        description="A shell command a verifier can run to confirm correctness (exit 0 on success)."
+    )
+
+
+@_hidden
+def _problem_statement(task_input: dict) -> str:
+    """Extract the task text from supported Harbor/benchmark field names."""
+    for key in ("user_message", "problem_statement", "task_description"):
+        value = task_input.get(key)
+        if isinstance(value, str) and value.strip():
+            return value.strip()
+    raise ValueError(
+        "task_input must include a non-empty user_message, problem_statement, or task_description"
+    )
+
+
+class BenchAgent(
+    Agent,
+    llm=FakeLLMClient(),
+    context={
+        "context_usage": DynamicContext(expr="self._context_usage_block()"),
+        "todo_status": DynamicContext(expr="self.todo.status()"),
+        "task": DynamicContext(expr="self.problem_statement"),
+    },
+):
+    """Generic agent for code and system tasks in containers.
+
+    ## Tools
+
+    ```python
+    r = await self.shell.run("command")                     # persistent shell
+    r = await self.shell.read("file.py", lines=(1,50))      # view -> Match
+    defs = await self.repo.symbols("src/", query="Handler") # definitions -> Match anchors
+    refs = await self.repo.refs("Handler", path="src/")     # usages -> Match anchors
+    await self.shell.replace(defs[0], new_code)              # edit at Match
+    await self.shell.write_file("file.py", content)         # create/overwrite
+    ```
+
+    ## Workflow
+
+    1. **Understand** -- explore the codebase/environment, reproduce the issue
+    2. **Plan** -- write a plan based on todos
+    3. **Implement and verify** -- make the fix and run relevant tests
+    4. **Return** -- ``return_result(TaskResult(...))`` with evidence
+
+    ## Return format
+
+    When you are done, you MUST return a ``TaskResult``:
+
+    ```python
+    return_result(TaskResult(
+        solution_description="Root cause: missing URL-encoding in auth.py. Fixed with quote_plus().",
+        evidence="pytest tests/test_login.py passed (3 passed in 0.4s)",
+        command_to_verify="pytest tests/test_login.py -x",
+    ))
+    ```
+
+    Use ``self.todo`` to track progress on multi-step tasks.
+    Mark todos done as you complete them.
+    """
+
+    shell: ShellTools
+    repo: RepoTools
+
+    def _context_usage_block(self) -> str:
+        """Return context-window usage plus a benchmark-agent compaction hint."""
+        if not self.context_stats:
+            return ""
+        return (
+            f"{self.context_stats.format()}\n"
+            "If event history is taking too much space, summarize older work "
+            "and call self.events.collapse(start_tag, end_tag, summary_text) "
+            "to replace it with a compact summary while keeping details accessible."
+        )
+
+    def __init__(self, llm: UnifiedLLM | None = None, **kwargs: Any) -> None:
+        super().__init__(llm=llm, **kwargs)
+        cwd = next((d for d in ("/testbed", "/app") if os.path.isdir(d)), os.getcwd())
+        self._install_python_tools(cwd)
+        self.todo = TodoManager()
+        self._seed_todos()
+        self.problem_statement = ""
+        # Base Agent hides context/events by default; BenchAgent's context_usage
+        # hint references them, so expose both APIs to the LLM here.
+        spec(self, "context", hidden=False)
+        spec(self, "events", hidden=False)
+        from nooa import Context
+
+        self.context_manager["python_tools"] = Context(doc(RepoTools, ShellTools), prefix=True)
+        self.context_manager["todo"] = Context(doc(type(self.todo)), prefix=True)
+
+    def _install_python_tools(self, cwd: str) -> None:
+        """Install shell/repo tools rooted at the same working directory."""
+        self.shell = ShellTools(cwd=cwd, init_command=_OPTIONAL_TESTBED_ACTIVATE)
+        self.repo = RepoTools(root=cwd, session=self.shell.session)
+
+    def _seed_todos(self) -> None:
+        """Preload the planning todo every benchmark task should start from."""
+        self.todo.add("Create a todo-based plan with clear dependencies")
+
+    async def _run_evaluation(self, task_input: dict) -> dict:
+        """Entry point called by the Harbor runner."""
+        # Read task fields generically (Harbor adapters vary in field names).
+        self.problem_statement = _problem_statement(task_input)
+        instructions = task_input.get("system_prompt") or task_input.get("instructions") or ""
+        initial_obs = task_input.get("initial_observation") or ""
+
+        if instructions:
+            self.context["instructions"] = instructions
+        if initial_obs:
+            self.context["initial_observation"] = initial_obs
+
+        # Reset shell to the task working dir.
+        cwd = task_input.get("working_dir")
+        if cwd:
+            if not os.path.isdir(cwd):
+                raise ValueError(f"working_dir does not exist: {cwd!r}")
+        else:
+            cwd = next((d for d in ("/testbed", "/app") if os.path.isdir(d)), os.getcwd())
+        self._install_python_tools(cwd)
+        from nooa import Context
+
+        self.context_manager["todo"] = Context(doc(type(self.todo)), prefix=True)
+        self.todo.clear()
+        self._seed_todos()
+
+        try:
+            result = await self._solve_task(self.problem_statement)
+            if isinstance(result, TaskResult):
+                return {
+                    "response": result.command_to_verify,
+                    "success": bool(result.solution_description),
+                    "result": result.model_dump(),
+                }
+            # Fallback for non-structured returns
+            result_str = str(result) if result is not None else ""
+            return {"response": result_str, "success": True, "result": result}
+        except Exception as e:
+            _logger.error("BenchAgent failed: %s", e)
+            return {"response": "", "success": False, "error": str(e)}
+
+    @strategy(
+        CodeActStrategy(
+            config=CodeActConfig(
+                max_iterations=300, max_retries=10, text_only_stop_behavior="synthetic_comment"
+            )
+        )
+    )
+    async def _solve_task(self, description: str) -> TaskResult:
+        """Solve the task.
+
+        You are an expert software engineer and system administrator working
+        inside a Linux container. Solve the task described below.
+
+        ## Task
+        {description}
+
+        ## Instructions
+        - Use ``await self.shell.run("command")`` to run shell commands.
+        - Use ``await self.shell.read("path")`` to view files.
+        - Use ``await self.repo.symbols(path, query="...")`` to find definitions.
+        - Use ``await self.repo.refs(name, path=".")`` to find usages.
+        - Use ``await self.shell.replace(...)`` to edit files or RepoTools matches.
+        - Use ``await self.shell.write_file(path, content)`` to create files.
+        - Use ``self.todo`` to track progress on multi-step work.
+        - You have root access; install packages as needed.
+        - Read task instructions carefully -- grading is strict and automated.
+
+        ## Verification & Return
+
+        Before finishing, run the relevant tests to confirm your work is correct.
+        Then return a structured result explaining WHY you believe the task is done:
+
+        ```python
+        return_result(TaskResult(
+            solution_description="The login handler didn't escape special chars in emails. Fixed by adding quote_plus() in auth.py:42.",
+            evidence="pytest tests/test_login.py -x passed: 5 passed in 1.2s",
+            command_to_verify="pytest tests/test_login.py -x",
+        ))
+        ```
+
+        ## Workflow
+
+        1. Explore and understand the task/codebase
+        2. Write a plan based on todos
+        3. Implement the solution and run tests to verify
+        4. Return ``TaskResult(...)`` with concrete evidence
+        """
+        ...

nooa_bench/protocol.py

@@ -0,0 +1,191 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+"""
+Core protocol definitions for the task execution infrastructure.
+
+This module defines the execution-engine and trace-analysis protocols along with
+the usage-statistics data structures shared across the evaluation runner.
+"""
+
+from collections.abc import Awaitable, Callable
+from dataclasses import dataclass, field
+from typing import Any, Protocol, TypeVar
+
+# ===========================
+# Layer 0/1/2 Protocols
+# ===========================
+# These protocols define the lower-level execution and analysis infrastructure
+# that the task runner builds on.
+
+R = TypeVar("R", covariant=True)
+
+
+@dataclass
+class EngineConfig:
+    """Configuration for execution engines (Layer 0).
+
+    Generic configuration that all execution engines understand.
+    Engine-specific configs can extend this.
+    """
+
+    max_concurrent: int = 10
+    timeout_seconds: float | None = None
+    enable_checkpointing: bool = True
+
+
+@dataclass
+class TaskState:
+    """Checkpoint state for a single task.
+
+    Used by execution engines to resume from crashes or skip completed tasks.
+    """
+
+    task_id: str
+    completed: bool
+    result: Any | None = None
+    error: str | None = None
+    timestamp: float | None = None
+
+
+class ExecutionEngine(Protocol):
+    """Protocol for swappable execution engines (Layer 0).
+
+    Different implementations provide different execution strategies:
+    - AsyncIOEngine: Async I/O with semaphore (for I/O-bound tasks like LLM APIs)
+    - MultiprocessEngine: Process pool (for CPU-bound tasks like local models)
+    - RayEngine: Distributed execution across cluster (for massive scale)
+    - NemoRunEngine: Slurm/cloud batch submission (for HPC clusters)
+
+    All engines share the same interface, allowing Layer 2 (Runner) to be
+    agnostic to the execution strategy.
+    """
+
+    async def run_tasks(
+        self,
+        task_ids: list[str],
+        task_fn: Callable[[str], Awaitable[R]],
+        config: EngineConfig,
+        checkpoint_state: list[TaskState] | None = None,
+        on_task_complete: Callable[[str, R | Exception], None] | None = None,
+    ) -> list[R]:
+        """Execute tasks using engine-specific strategy.
+
+        Args:
+            task_ids: List of task identifiers
+            task_fn: Async function that takes task_id and returns result
+            config: Engine configuration
+            checkpoint_state: Previous run state for resume (optional)
+            on_task_complete: Callback for each completed task (optional)
+
+        Returns:
+            List of results in same order as task_ids
+        """
+        ...
+
+
+@dataclass
+class ModelUsageStats:
+    """Usage statistics for a single model."""
+
+    model_name: str
+    prompt_tokens: int = 0
+    completion_tokens: int = 0
+    total_tokens: int = 0
+    call_count: int = 0
+    latencies_ms: list[float] = field(default_factory=list)
+
+    @property
+    def p95_latency_ms(self) -> float | None:
+        """95th percentile latency in milliseconds."""
+        if not self.latencies_ms:
+            return None
+        sorted_latencies = sorted(self.latencies_ms)
+        idx = int(len(sorted_latencies) * 0.95)
+        return sorted_latencies[idx]
+
+
+@dataclass
+class TaskUsageStats:
+    """Usage statistics extracted from a single task's trace."""
+
+    task_id: str
+    total_runtime_seconds: float
+    models_used: list[ModelUsageStats] = field(default_factory=list)
+    total_llm_calls: int = 0
+
+    @property
+    def total_tokens(self) -> int:
+        """Total tokens across all models."""
+        return sum(m.total_tokens for m in self.models_used)
+
+    @property
+    def total_prompt_tokens(self) -> int:
+        """Total prompt tokens across all models."""
+        return sum(m.prompt_tokens for m in self.models_used)
+
+    @property
+    def total_completion_tokens(self) -> int:
+        """Total completion tokens across all models."""
+        return sum(m.completion_tokens for m in self.models_used)
+
+
+@dataclass
+class AggregateUsageStats:
+    """Aggregate usage statistics across multiple tasks."""
+
+    num_tasks: int
+    per_task_stats: list[TaskUsageStats] = field(default_factory=list)
+
+    @property
+    def total_tokens(self) -> int:
+        """Total tokens across all tasks."""
+        return sum(s.total_tokens for s in self.per_task_stats)
+
+    @property
+    def total_runtime_seconds(self) -> float:
+        """Total runtime across all tasks."""
+        return sum(s.total_runtime_seconds for s in self.per_task_stats)
+
+    @property
+    def models_breakdown(self) -> dict[str, ModelUsageStats]:
+        """Aggregate stats per model across all tasks."""
+        by_model: dict[str, ModelUsageStats] = {}
+
+        for task_stats in self.per_task_stats:
+            for model_stats in task_stats.models_used:
+                if model_stats.model_name not in by_model:
+                    by_model[model_stats.model_name] = ModelUsageStats(
+                        model_name=model_stats.model_name
+                    )
+
+                agg = by_model[model_stats.model_name]
+                agg.prompt_tokens += model_stats.prompt_tokens
+                agg.completion_tokens += model_stats.completion_tokens
+                agg.total_tokens += model_stats.total_tokens
+                agg.call_count += model_stats.call_count
+                agg.latencies_ms.extend(model_stats.latencies_ms)
+
+        return by_model
+
+
+class TraceAnalyzer(Protocol):
+    """Protocol for analyzing OTel traces to extract usage statistics.
+
+    Implementations read .jsonl trace files and extract:
+    - Token counts per model
+    - LLM call latencies
+    - Total runtime
+
+    This avoids duplicating trace data in IntermediateSteps.
+    """
+
+    def analyze_trace(self, trace_path: str) -> TaskUsageStats:
+        """Analyze a single trace file and extract usage statistics.
+
+        Args:
+            trace_path: Path to .jsonl trace file
+
+        Returns:
+            TaskUsageStats with extracted metrics
+        """
+        ...

nooa_bench/runner.py

@@ -0,0 +1,235 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+"""
+Harbor agent runner for NOOA — executed inside the Harbor container.
+
+Harbor invokes this as a CLI process::
+
+    nemo-harbor \\
+        --instruction '...' \\
+        --model 'anthropic/claude-opus-4-6' \\
+        --agent-type bench
+
+The runner is benchmark-agnostic. Its only jobs are:
+
+1. Instantiate the right agent class from ``--agent-type``.
+2. Call ``agent._run_evaluation({"user_message": instruction})``.
+3. Write ``result.json`` to ``/logs/agent/`` and answer text to ``/app/answer.txt``.
+
+Benchmark-specific logic (system prompts, instruction parsing, data paths) lives
+inside each agent class, not here.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import importlib
+import json
+import logging
+import os
+import sys
+from pathlib import Path
+from typing import Any
+
+import click
+
+logger = logging.getLogger("nooa_bench.runner")
+
+# Harbor container path conventions
+LOGS_DIR = Path("/logs/agent")
+ARTIFACTS_DIR = Path("/logs/artifacts")
+TRACES_DIR = ARTIFACTS_DIR / "traces"
+ANSWER_FILE = Path("/app/answer.txt")
+
+
+def _setup_logging() -> None:
+    handlers: list[logging.Handler] = [logging.StreamHandler(sys.stderr)]
+    try:
+        LOGS_DIR.mkdir(parents=True, exist_ok=True)
+        handlers.append(logging.FileHandler(str(LOGS_DIR / "nooa_bench.log")))
+    except OSError:
+        # Outside a Harbor container /logs may not exist or be writable — stderr only.
+        pass
+    logging.basicConfig(
+        level=logging.INFO,
+        format="%(asctime)s [%(name)s] %(levelname)s: %(message)s",
+        handlers=handlers,
+    )
+
+
+def _setup_tracing(model: str, agent_type: str) -> None:
+    """Enable OTel tracing, publishing live to the viewer if reachable.
+
+    Detection order:
+    1. Probe ``OTLP_ENDPOINT`` env var, or ``http://localhost:5001`` by default.
+    2. If reachable → stream spans via the journal exporter with ``eval.model``
+       and ``eval.agent_type`` injected as resource attributes so the viewer can
+       display them without a separate ``import-harbor`` step.
+    3. If unreachable → fall back to JSONL files in the Harbor artifact
+       directory (``/logs/artifacts/traces/``), importable later via
+       ``nemo-oo import-harbor``.
+
+    Note: Apptainer containers share the host network namespace, so
+    ``localhost:5001`` inside the container resolves to the developer's host.
+    For Docker containers set ``OTLP_ENDPOINT=http://host.docker.internal:5001``.
+    """
+    try:
+        from nooa.tracing import (
+            enable_tracing,
+            probe_otlp_endpoint,
+        )
+        from nooa.tracing import exporters as nemo_exporters
+    except ImportError:
+        logger.warning("nooa.tracing not available, no tracing")
+        return
+
+    endpoint = os.environ.get("OTLP_ENDPOINT", "http://localhost:5001/v1/traces")
+
+    if probe_otlp_endpoint(endpoint):
+        logger.info("OTLP endpoint reachable (%s) — streaming traces live", endpoint)
+        enable_tracing(
+            exporters=[nemo_exporters.journal(endpoint=endpoint)],
+            extra_resource_attrs={"eval.model": model, "eval.agent_type": agent_type},
+        )
+        logger.info("OTel tracing enabled → %s", endpoint)
+    else:
+        TRACES_DIR.mkdir(parents=True, exist_ok=True)
+        enable_tracing(exporters=[nemo_exporters.jsonl(TRACES_DIR)])
+        logger.info("OTel tracing enabled → %s (OTLP not reachable)", TRACES_DIR)
+
+
+def _import_agent_class(agent_type: str) -> type:
+    from nooa_bench import AGENT_CLASSES
+
+    entry = AGENT_CLASSES.get(agent_type)
+    if entry is None:
+        raise ValueError(
+            f"Unknown agent_type: {agent_type!r}.  Must be one of: {sorted(AGENT_CLASSES)}"
+        )
+    module_path, class_name = entry.rsplit(":", 1)
+    mod = importlib.import_module(module_path)
+    return getattr(mod, class_name)
+
+
+def _write_result(result: dict[str, Any], model: str, agent_type: str) -> None:
+    """Write result metadata to LOGS_DIR/result.json."""
+    LOGS_DIR.mkdir(parents=True, exist_ok=True)
+    payload = {
+        "model": model,
+        "agent_type": agent_type,
+        "success": result.get("success", False),
+        "response": result.get("response", ""),
+        "error": result.get("error"),
+        "n_input_tokens": result.get("n_input_tokens"),
+        "n_output_tokens": result.get("n_output_tokens"),
+    }
+    out = LOGS_DIR / "result.json"
+    out.write_text(json.dumps(payload, indent=2))
+    logger.info("Result written → %s", out)
+
+
+def _write_answer(result: dict[str, Any]) -> None:
+    """Write the agent's answer to /app/answer.txt for Harbor's verifier."""
+    answer = result.get("answer") or result.get("response", "")
+    if not answer:
+        logger.warning("No answer to write to %s", ANSWER_FILE)
+        return
+    try:
+        ANSWER_FILE.parent.mkdir(parents=True, exist_ok=True)
+        ANSWER_FILE.write_text(str(answer))
+        logger.info("Answer written → %s", ANSWER_FILE)
+    except OSError as e:
+        logger.warning("Could not write answer file %s: %s", ANSWER_FILE, e)
+
+
+async def _run(
+    instruction: str,
+    model: str,
+    agent_type: str,
+    api_base: str | None,
+    working_dir: str | None = None,
+) -> int:
+    """Async main: instantiate, wire, run.  Returns exit code (0 = success)."""
+    from nooa.unifiedllm import get_llm_client
+
+    # Build LLM client — honour env-var overrides for local vLLM deployments.
+    llm_overrides: dict[str, str] = {}
+    if api_base:
+        llm_overrides["api_base"] = api_base
+    elif base_url := os.environ.get("OPENAI_BASE_URL"):
+        llm_overrides["api_base"] = base_url
+    if api_key := os.environ.get("OPENAI_API_KEY"):
+        llm_overrides["api_key"] = api_key
+
+    llm_client = get_llm_client(model, **llm_overrides)
+
+    # Instantiate agent.
+    AgentClass = _import_agent_class(agent_type)
+    agent: Any = AgentClass(llm=llm_client)
+
+    # All agents share the same interface: {"user_message": instruction}.
+    # Benchmark-specific parsing (system prompts, data paths, etc.) happens
+    # inside the agent's _run_evaluation method.
+    from nooa.runtime.token_usage import get_task_tokens, start_task_tokens
+
+    logger.info("Running agent %s (model=%s)...", agent_type, model)
+    start_task_tokens()
+    task_input: dict[str, Any] = {"user_message": instruction}
+    if working_dir:
+        task_input["working_dir"] = working_dir
+    result = await agent._run_evaluation(task_input)
+    result.update(get_task_tokens())
+    _write_result(result, model, agent_type)
+    _write_answer(result)
+
+    if result.get("success"):
+        logger.info("Agent completed successfully.")
+        return 0
+    else:
+        logger.error("Agent reported failure.")
+        return 1
+
+
+@click.command()
+@click.option("--instruction", required=True, help="Task instruction / problem statement")
+@click.option("--model", required=True, help="Model name in litellm format")
+@click.option("--agent-type", default="bench", show_default=True, help="Agent variant to run")
+@click.option("--working-dir", default=None, help="Working directory for the agent shell session")
+@click.option("--api-base", default=None, help="Override API base URL")
+def main(
+    instruction: str,
+    model: str,
+    agent_type: str,
+    working_dir: str | None,
+    api_base: str | None,
+) -> None:
+    """Run a NOOA agent on a task inside a Harbor container."""
+    _setup_logging()
+    logger.info("nooa-bench runner starting")
+    logger.info("  model:      %s", model)
+    logger.info("  agent_type: %s", agent_type)
+    if api_base:
+        logger.info("  api_base:   %s", api_base)
+
+    # Validate agent_type early so we get a clean error before any heavy imports.
+    from nooa_bench import AGENT_CLASSES
+
+    if agent_type not in AGENT_CLASSES:
+        logger.error(
+            "Unknown agent_type: %r.  Must be one of: %s", agent_type, sorted(AGENT_CLASSES)
+        )
+        sys.exit(1)
+
+    _setup_tracing(model=model, agent_type=agent_type)
+
+    try:
+        exit_code = asyncio.run(_run(instruction, model, agent_type, api_base, working_dir))
+    except Exception as e:
+        logger.exception("Runner failed with unhandled exception: %s", e)
+        exit_code = 1
+
+    sys.exit(exit_code)
+
+
+if __name__ == "__main__":
+    main()

nooa_bench/trace_analyzer.py

@@ -0,0 +1,186 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+"""
+Trace analyzer for extracting usage statistics from execution traces.
+
+This module reads JSONL trace files produced by the agent framework and
+extracts per-model token counts, LLM call latencies, and total runtime.
+"""
+
+import json
+import logging
+from pathlib import Path
+from typing import Any
+
+from nooa_bench.protocol import (
+    ModelUsageStats,
+    TaskUsageStats,
+)
+
+logger = logging.getLogger(__name__)
+
+
+class TraceAnalyzer:
+    """
+    Analyzes OTel traces to extract usage statistics.
+
+    Implements the TraceAnalyzer protocol: reads .jsonl trace files and
+    extracts token counts per model, LLM call latencies, and total runtime.
+    """
+
+    def analyze_trace(self, trace_path: str) -> TaskUsageStats:
+        """
+        Analyze a trace file and extract usage statistics.
+
+        Implements the TraceAnalyzer protocol to extract:
+        - Token counts per model
+        - LLM call latencies
+        - Total runtime
+        - Model usage breakdown
+
+        Args:
+            trace_path: Path to .jsonl trace file
+
+        Returns:
+            TaskUsageStats with extracted metrics
+        """
+        path = Path(trace_path)
+        if not path.exists():
+            # Return empty stats for missing trace
+            return TaskUsageStats(
+                task_id=path.stem,
+                total_runtime_seconds=0.0,
+                models_used=[],
+                total_llm_calls=0,
+            )
+
+        # Track stats per model
+        model_stats: dict[str, ModelUsageStats] = {}
+        trace_start_time: float | None = None
+        trace_end_time: float | None = None
+        total_llm_calls = 0
+
+        with open(path) as f:
+            for line in f:
+                try:
+                    span = json.loads(line.strip())
+
+                    # Track overall trace timing
+                    start_time = span.get("start_time_unix_nano", 0) / 1e9
+                    end_time = span.get("end_time_unix_nano", 0) / 1e9
+
+                    if trace_start_time is None or start_time < trace_start_time:
+                        trace_start_time = start_time
+                    if trace_end_time is None or end_time > trace_end_time:
+                        trace_end_time = end_time
+
+                    # Look for LLM spans
+                    attrs = span.get("attributes", {})
+                    span_name = span.get("name", "")
+
+                    # Check if this is an LLM call span
+                    if self._is_llm_span(span_name, attrs):
+                        total_llm_calls += 1
+
+                        # Extract model name
+                        model_name = attrs.get(
+                            "llm.model", attrs.get("gen_ai.request.model", "unknown")
+                        )
+
+                        # Initialize model stats if not seen before
+                        if model_name not in model_stats:
+                            model_stats[model_name] = ModelUsageStats(model_name=model_name)
+
+                        stats = model_stats[model_name]
+
+                        # Extract token counts with validation
+                        prompt_tokens = attrs.get(
+                            "llm.token_count.prompt", attrs.get("gen_ai.usage.input_tokens", 0)
+                        )
+                        completion_tokens = attrs.get(
+                            "llm.token_count.completion", attrs.get("gen_ai.usage.output_tokens", 0)
+                        )
+
+                        # Validate and parse token counts
+                        prompt_count = self._parse_token_count(prompt_tokens, "prompt_tokens", path)
+                        completion_count = self._parse_token_count(
+                            completion_tokens, "completion_tokens", path
+                        )
+
+                        stats.prompt_tokens += prompt_count
+                        stats.completion_tokens += completion_count
+                        stats.total_tokens += prompt_count + completion_count
+                        stats.call_count += 1
+
+                        # Calculate latency in milliseconds
+                        if start_time > 0 and end_time > 0:
+                            latency_ms = (end_time - start_time) * 1000
+                            stats.latencies_ms.append(latency_ms)
+
+                except json.JSONDecodeError:
+                    continue
+
+        # Calculate total runtime
+        runtime_seconds = 0.0
+        if trace_start_time and trace_end_time:
+            runtime_seconds = trace_end_time - trace_start_time
+
+        return TaskUsageStats(
+            task_id=path.stem,
+            total_runtime_seconds=runtime_seconds,
+            models_used=list(model_stats.values()),
+            total_llm_calls=total_llm_calls,
+        )
+
+    def _is_llm_span(self, span_name: str, attributes: dict[str, Any]) -> bool:
+        """Check if a span represents an LLM API call."""
+        # Check span name
+        llm_span_names = ["llm", "chat", "completion", "generation", "inference"]
+        if any(name in span_name.lower() for name in llm_span_names):
+            return True
+
+        # Check for LLM-related attributes
+        llm_attrs = [
+            "llm.model",
+            "llm.token_count.prompt",
+            "llm.token_count.completion",
+            "gen_ai.request.model",
+            "gen_ai.usage.input_tokens",
+            "gen_ai.usage.output_tokens",
+        ]
+        return any(attr in attributes for attr in llm_attrs)
+
+    def _parse_token_count(self, value: Any, field_name: str, trace_path: Path) -> int:
+        """Parse a token count value with validation.
+
+        Args:
+            value: The raw value from the trace (could be int, str, None, etc.)
+            field_name: Name of the field for logging
+            trace_path: Path to trace file for logging context
+
+        Returns:
+            Parsed integer token count, or 0 if invalid
+        """
+        if value is None:
+            return 0
+
+        if isinstance(value, int):
+            return value
+
+        if isinstance(value, float):
+            return int(value)
+
+        if isinstance(value, str):
+            try:
+                return int(value)
+            except ValueError:
+                logger.warning(
+                    f"Invalid {field_name} value '{value}' in trace {trace_path.name}, using 0"
+                )
+                return 0
+
+        logger.warning(
+            f"Unexpected type {type(value).__name__} for {field_name} in trace "
+            f"{trace_path.name}, using 0"
+        )
+        return 0

nooa_bench-0.0.7.dist-info/METADATA

@@ -0,0 +1,29 @@
+Metadata-Version: 2.4
+Name: nooa-bench
+Version: 0.0.7
+Summary: Benchmark agent (BenchAgent) and Harbor runner for the NOOA framework — reproduces the tech report's SWE-bench and Terminal-Bench results
+Project-URL: Homepage, https://github.com/NVIDIA-NeMo/labs-OO-Agents
+Project-URL: Repository, https://github.com/NVIDIA-NeMo/labs-OO-Agents
+Project-URL: Issues, https://github.com/NVIDIA-NeMo/labs-OO-Agents/issues
+License: Apache-2.0
+Requires-Python: <3.14,>=3.12
+Requires-Dist: click>=8.1
+Requires-Dist: nooa
+Requires-Dist: nooa-cli
+Description-Content-Type: text/markdown
+
+# nooa-bench
+
+Benchmark agent (`BenchAgent`) and Harbor runner for
+[NOOA](https://github.com/NVIDIA-NeMo/labs-OO-Agents). Reproduces the SWE-bench
+and Terminal-Bench results from the NOOA tech report.
+
+```bash
+uv add nooa-bench
+nemo-harbor --help
+```
+
+See the [main repository](https://github.com/NVIDIA-NeMo/labs-OO-Agents) for
+documentation.
+
+Apache-2.0 licensed.

nooa_bench-0.0.7.dist-info/RECORD

@@ -0,0 +1,9 @@
+nooa_bench/__init__.py,sha256=PnppDSvQWWJK2au5iCLEXXuT3EW57U7mNuuR8uDa_LU,776
+nooa_bench/bench_agent.py,sha256=XThiC8G-GSFEIu7b9iLP9WhhPxFpHoLJgK38rDxfOnA,10243
+nooa_bench/protocol.py,sha256=h3xbUHd986MXPXF2guQPtFSb1OU6aPdDId8QPcYHUS8,6076
+nooa_bench/runner.py,sha256=TkFUg6GdwmtL1N_eTPtrp0gcFgfXR9GQRT-K6wByKW0,8365
+nooa_bench/trace_analyzer.py,sha256=Fg6WXj_vJDcHiq8-WxW6Et5AzHs9c5eRqmwuJ7oDneA,6663
+nooa_bench-0.0.7.dist-info/METADATA,sha256=Dxy7TjFB0-regKpqn0rpKo8fb8wqgDLzGy-YT9_DGwQ,944
+nooa_bench-0.0.7.dist-info/WHEEL,sha256=mffPy8wBnZQn2VnJUU5jE99KsxaSfiyMHV9Yt0aLVxs,87
+nooa_bench-0.0.7.dist-info/entry_points.txt,sha256=3J9ysVhArjnuOz0ZjXMXXF6t-TSvqt5zJDOUEGmsFJ4,55
+nooa_bench-0.0.7.dist-info/RECORD,,

nooa_bench-0.0.7.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.30.1
+Root-Is-Purelib: true
+Tag: py3-none-any

nooa_bench-0.0.7.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+nemo-harbor = nooa_bench.runner:main