harborforge 1.0.0__py3-none-any.whl

harborforge/__init__.py

@@ -0,0 +1,6 @@
+"""harborforge — abstract toolkit for mapping benchmark datasets to Harbor task directories."""
+
+from .handlers.base import DatasetHandler
+from .mapper import DataMapper
+
+__all__ = ["DataMapper", "DatasetHandler"]

harborforge/handlers/__init__.py

@@ -0,0 +1,19 @@
+from .base import (
+    DatasetHandler,
+    case_insensitive_verifier,
+    exact_match_verifier,
+    list_verifier,
+    no_verifier,
+    numeric_verifier,
+    script_verifier,
+)
+
+__all__ = [
+    "DatasetHandler",
+    "case_insensitive_verifier",
+    "exact_match_verifier",
+    "list_verifier",
+    "no_verifier",
+    "numeric_verifier",
+    "script_verifier",
+]

harborforge/handlers/base.py

@@ -0,0 +1,231 @@
+"""
+DatasetHandler contract.
+
+Each dataset subclass must implement:
+  - instruction(task_data)  → str   content for instruction.md (shown to the agent, no answer leakage)
+  - test_sh(task_data)      → str   content for tests/test.sh (must write reward to /logs/verifier/reward.txt)
+  - dockerfile(task_data)   → str   content for environment/Dockerfile
+
+Optional overrides:
+  - data_files(task_data)   → list[tuple[Path, str]]  (local_path, dest_in_build_context) pairs
+  - task_toml(task_id)      → str   content for task.toml
+"""
+
+from abc import ABC, abstractmethod
+from pathlib import Path
+from typing import Any
+
+
+class DatasetHandler(ABC):
+    dataset_name: str
+
+    @abstractmethod
+    def instruction(self, task_data: dict[str, Any]) -> str:
+        """Returns instruction.md content — what the agent sees. Must not include the answer."""
+
+    @abstractmethod
+    def test_sh(self, task_data: dict[str, Any]) -> str:
+        """Returns tests/test.sh content. Must write a float reward to /logs/verifier/reward.txt."""
+
+    @abstractmethod
+    def dockerfile(self, task_data: dict[str, Any]) -> str:
+        """Returns environment/Dockerfile content."""
+
+    def verifier_dockerfile(self, task_data: dict[str, Any]) -> str | None:
+        """Return a Dockerfile for a SEPARATE verifier container, or None for SHARED (default).
+
+        When non-None, Harbor routes the trial through SEPARATE mode: the verifier runs in
+        its own isolated container and receives agent artifacts as input. Use this when the
+        verifier needs different dependencies or ground truth data isolated from the agent.
+        """
+        return None
+
+    def setup(self) -> None:
+        """Download or prepare local data required by this dataset. No-op by default."""
+
+    def artifacts(self) -> list[str]:
+        """Container paths to capture as artifacts after a trial. Empty by default."""
+        return []
+
+    def verifier_env_keys(self) -> list[str]:
+        """Environment variable keys to forward to the SEPARATE verifier container."""
+        return []
+
+    def data_files(self, task_data: dict[str, Any]) -> list[tuple[Path, str]]:
+        """
+        Returns (local_path, dest_in_build_context) pairs for files to COPY into the image.
+        dest_in_build_context is relative to environment/ (e.g. 'data/train.csv').
+        """
+        return []
+
+    def task_toml(self, task_id: str) -> str:
+        return f"""\
+version = "1.0"
+
+[task]
+name = "{task_id}"
+
+[metadata]
+
+[verifier]
+timeout_sec = 900.0
+
+[agent]
+timeout_sec = 900.0
+
+[environment]
+build_timeout_sec = 600.0
+"""
+
+
+# ---------------------------------------------------------------------------
+# Verifier shell script templates
+# ---------------------------------------------------------------------------
+
+
+def exact_match_verifier(answer: Any) -> str:
+    """Reward 1.0 if agent output matches expected answer (whitespace-stripped string comparison)."""
+    safe = str(answer).replace("'", "'\\''")
+    return f"""\
+#!/bin/bash
+set -e
+mkdir -p /logs/verifier
+
+EXPECTED='{safe}'
+SUBMISSION=/app/submission.txt
+
+if [ ! -f "$SUBMISSION" ]; then
+    echo "0" > /logs/verifier/reward.txt
+    exit 0
+fi
+
+ACTUAL=$(cat "$SUBMISSION" | tr -d '[:space:]')
+EXPECTED_CLEAN=$(echo "$EXPECTED" | tr -d '[:space:]')
+
+if [ "$ACTUAL" = "$EXPECTED_CLEAN" ]; then
+    echo "1" > /logs/verifier/reward.txt
+else
+    echo "0" > /logs/verifier/reward.txt
+fi
+"""
+
+
+def numeric_verifier(answer: Any, tolerance: float = 0.05) -> str:
+    """Reward 1.0 if agent output is within `tolerance` (relative) of expected numeric value."""
+    safe = str(answer).replace("'", "'\\''")
+    return f"""\
+#!/bin/bash
+set -e
+mkdir -p /logs/verifier
+
+EXPECTED='{safe}'
+
+python3 - "$EXPECTED" <<'PYEOF'
+import sys
+
+try:
+    expected = float(sys.argv[1])
+    with open("/app/submission.txt") as f:
+        actual = float(f.read().strip())
+    denom = abs(expected) if expected != 0 else 1.0
+    reward = 1.0 if abs(actual - expected) / denom <= {tolerance} else 0.0
+except Exception:
+    reward = 0.0
+
+with open("/logs/verifier/reward.txt", "w") as f:
+    f.write(str(reward))
+PYEOF
+"""
+
+
+def list_verifier(answer: Any) -> str:
+    """Reward 1.0 if agent output contains all expected items (order-insensitive, case-insensitive).
+    Expected answer should be a Python list repr or comma-separated string."""
+    safe = str(answer).replace("'", "'\\''")
+    return f"""\
+#!/bin/bash
+set -e
+mkdir -p /logs/verifier
+
+python3 <<'PYEOF'
+import ast, re
+
+EXPECTED_RAW = '{safe}'
+try:
+    with open("/app/submission.txt") as f:
+        actual_raw = f.read().strip()
+except FileNotFoundError:
+    open("/logs/verifier/reward.txt", "w").write("0")
+    raise SystemExit
+
+def parse_list(s):
+    try:
+        val = ast.literal_eval(s)
+        if isinstance(val, list):
+            return {{str(x).strip().lower() for x in val}}
+    except Exception:
+        pass
+    return {{x.strip().lower() for x in re.split(r"[,\\n]+", s) if x.strip()}}
+
+expected = parse_list(EXPECTED_RAW)
+actual = parse_list(actual_raw)
+reward = 1.0 if expected and expected == actual else 0.0
+open("/logs/verifier/reward.txt", "w").write(str(reward))
+PYEOF
+"""
+
+
+def case_insensitive_verifier(answer: Any) -> str:
+    """Reward 1.0 if agent output matches expected answer, ignoring case and whitespace."""
+    safe = str(answer).replace("'", "'\\''")
+    return f"""\
+#!/bin/bash
+set -e
+mkdir -p /logs/verifier
+
+EXPECTED='{safe}'
+SUBMISSION=/app/submission.txt
+
+if [ ! -f "$SUBMISSION" ]; then
+    echo "0" > /logs/verifier/reward.txt
+    exit 0
+fi
+
+ACTUAL=$(cat "$SUBMISSION" | tr '[:upper:]' '[:lower:]' | tr -d '[:space:]')
+EXPECTED_CLEAN=$(echo "$EXPECTED" | tr '[:upper:]' '[:lower:]' | tr -d '[:space:]')
+
+if [ "$ACTUAL" = "$EXPECTED_CLEAN" ]; then
+    echo "1" > /logs/verifier/reward.txt
+else
+    echo "0" > /logs/verifier/reward.txt
+fi
+"""
+
+
+def script_verifier(script_container_path: str) -> str:
+    """Runs a Python eval script already present in the container.
+    The script is fully responsible for writing a float reward to /logs/verifier/reward.txt."""
+    return f"""\
+#!/bin/bash
+set -e
+mkdir -p /logs/verifier
+
+SCRIPT="{script_container_path}"
+
+if [ ! -f "$SCRIPT" ]; then
+    echo "0" > /logs/verifier/reward.txt
+    exit 0
+fi
+
+python3 "$SCRIPT" || echo "0" > /logs/verifier/reward.txt
+"""
+
+
+def no_verifier(reason: str = "External evaluation required") -> str:
+    """Always writes reward 0. Used for tasks that require external scoring (e.g. leaderboards)."""
+    return f"""\
+#!/bin/bash
+# {reason}
+mkdir -p /logs/verifier
+echo "0" > /logs/verifier/reward.txt
+"""

harborforge/mapper.py

@@ -0,0 +1,142 @@
+"""
+DataMapper — abstract base for benchmark → Harbor task directory pipelines.
+
+Subclass and implement `iter_tasks()`. Override `setup()` to handle data
+downloads or prep. Call `run()` as the standard entry point, or `map()`
+directly for more control.
+"""
+
+import shutil
+import stat
+from abc import ABC, abstractmethod
+from collections.abc import Iterator
+from pathlib import Path
+from typing import Any
+
+from .handlers.base import DatasetHandler
+
+
+class DataMapper(ABC):
+    """
+    Abstract mapper from a benchmark dataset to Harbor task directories.
+
+    Minimal adapter implementation:
+      1. Implement `iter_tasks()` to yield tasks.
+      2. Optionally override `setup()` to download/prepare raw data.
+      3. Call `run()` as the entry point — it handles setup, mapping, and registry.
+    """
+
+    # ---------------------------------------------------------------------------
+    # Public API
+    # ---------------------------------------------------------------------------
+
+    def setup(self) -> None:
+        """Download or prepare raw data before mapping. No-op by default."""
+
+    def map(self, output_dir: Path, registry_path: Path | None = None) -> int:
+        """
+        Write all tasks to output_dir as Harbor task directories.
+        Wipes and recreates output_dir on each call.
+        If registry_path is given, auto-generates a Harbor registry.json.
+        Returns the number of tasks written.
+        """
+        if output_dir.exists():
+            shutil.rmtree(output_dir)
+
+        total = 0
+        current_dataset = ""
+        for task_id, dir_name, handler, task_data in self.iter_tasks():
+            self._write_task(output_dir / dir_name, task_id, handler, task_data)
+            total += 1
+            dataset = dir_name.split("/")[0]
+            if dataset != current_dataset:
+                if current_dataset:
+                    print()
+                current_dataset = dataset
+                print(f"  ↳ {dataset}", end="", flush=True)
+            print(f"\r  ↳ {dataset} ({total} written)", end="", flush=True)
+        if total:
+            print()
+
+        if registry_path is not None:
+            from harborforge.registry import generate_registry
+
+            counts = generate_registry(output_dir, registry_path)
+            n_datasets = len(counts)
+            n_tasks = sum(counts.values())
+            print(f"✅ {registry_path} — {n_tasks} tasks across {n_datasets} datasets")
+
+        return total
+
+    def run(self, output_dir: Path, registry_path: Path | None = None) -> None:
+        """
+        Standard entry point: setup() → map() → optional registry.
+        Adapters call this from __main__.py.
+        """
+        self.setup()
+        total = self.map(output_dir, registry_path)
+        print(f"📊 Total tasks written: {total}")
+
+    # ---------------------------------------------------------------------------
+    # Abstract
+    # ---------------------------------------------------------------------------
+
+    @abstractmethod
+    def iter_tasks(self) -> Iterator[tuple[str, str, DatasetHandler, dict[str, Any]]]:
+        """
+        Yield (task_id, dir_name, handler, raw_task_data) for each task.
+
+        - task_id:   unique identifier used in task.toml name field
+        - dir_name:  relative path under output_dir (e.g. 'daeval/0')
+        - handler:   DatasetHandler instance for this dataset
+        - raw_task_data: raw dict from the source benchmark file
+        """
+
+    # ---------------------------------------------------------------------------
+    # Internal
+    # ---------------------------------------------------------------------------
+
+    def _write_task(
+        self,
+        task_dir: Path,
+        task_id: str,
+        handler: DatasetHandler,
+        task_data: dict[str, Any],
+    ) -> None:
+        task_dir.mkdir(parents=True, exist_ok=True)
+
+        (task_dir / "instruction.md").write_text(handler.instruction(task_data), encoding="utf-8")
+
+        env_dir = task_dir / "environment"
+        env_dir.mkdir(exist_ok=True)
+
+        for local_path, dest_name in handler.data_files(task_data):
+            dest = env_dir / dest_name
+            dest.parent.mkdir(parents=True, exist_ok=True)
+            shutil.copy2(local_path, dest)
+
+        (env_dir / "Dockerfile").write_text(handler.dockerfile(task_data), encoding="utf-8")
+
+        # Auto-detect verifier mode: SEPARATE if handler provides a verifier Dockerfile.
+        # Harbor uses tests/ as the verifier build context in SEPARATE mode.
+        task_toml = handler.task_toml(task_id)
+        verifier_df = handler.verifier_dockerfile(task_data)
+        if verifier_df is not None:
+            task_toml += "\n[verifier.environment]\nbuild_timeout_sec = 300.0\n"
+
+        (task_dir / "task.toml").write_text(task_toml, encoding="utf-8")
+
+        tests_dir = task_dir / "tests"
+        tests_dir.mkdir(exist_ok=True)
+        test_sh = tests_dir / "test.sh"
+        test_sh.write_text(handler.test_sh(task_data), encoding="utf-8")
+        test_sh.chmod(test_sh.stat().st_mode | stat.S_IEXEC | stat.S_IXGRP | stat.S_IXOTH)
+
+        # SEPARATE mode: write verifier Dockerfile into tests/ (Harbor's verifier build context).
+        # COPY test.sh into the image so Harbor can execute /tests/test.sh inside the container.
+        if verifier_df is not None:
+            verifier_df_with_copy = (
+                verifier_df.rstrip()
+                + "\nCOPY test.sh /tests/test.sh\nRUN chmod +x /tests/test.sh\n"
+            )
+            (tests_dir / "Dockerfile").write_text(verifier_df_with_copy, encoding="utf-8")

harborforge/registry.py

@@ -0,0 +1,61 @@
+"""
+Generate a local Harbor registry.json from a directory of Harbor task directories.
+
+The tasks directory is expected to have the structure:
+    tasks_dir/
+        dataset_a/
+            task_0/
+            task_1/
+        dataset_b/
+            task_0/
+
+Each top-level subdirectory becomes a named dataset in the registry.
+"""
+
+import json
+from pathlib import Path
+
+
+def generate_registry(
+    tasks_dir: Path,
+    output_path: Path,
+    version: str = "1.0",
+    description_template: str = "{name} benchmark tasks",
+) -> dict[str, int]:
+    """
+    Scan tasks_dir and write a Harbor registry.json to output_path.
+
+    Returns a dict of {dataset_name: task_count}.
+    """
+    if not tasks_dir.exists():
+        raise FileNotFoundError(f"Tasks directory not found: {tasks_dir}")
+
+    datasets = []
+    counts: dict[str, int] = {}
+
+    for dataset_dir in sorted(tasks_dir.iterdir()):
+        if not dataset_dir.is_dir():
+            continue
+
+        tasks = [
+            {"name": task.name, "path": str(task)}
+            for task in sorted(dataset_dir.iterdir())
+            if task.is_dir()
+        ]
+
+        if not tasks:
+            continue
+
+        name = dataset_dir.name.lower()
+        datasets.append(
+            {
+                "name": name,
+                "version": version,
+                "description": description_template.format(name=dataset_dir.name),
+                "tasks": tasks,
+            }
+        )
+        counts[name] = len(tasks)
+
+    output_path.write_text(json.dumps(datasets, indent=2))
+    return counts

harborforge/run.py

@@ -0,0 +1,117 @@
+"""
+Generic Harbor trial runner.
+
+Wraps `harbor jobs start` with handler-aware artifact/credential forwarding.
+Adapters call `run()` with their registry, tasks dir, and defaults.
+"""
+
+import argparse
+import os
+import subprocess
+import sys
+import time
+from pathlib import Path
+from typing import Any
+
+
+def _is_single_task(task: str) -> bool:
+    """True when task is a specific task (dataset/id), False when it's a dataset name."""
+    return "/" in task
+
+
+def run(
+    handler_registry: dict[str, Any],
+    *,
+    tasks_dir: Path,
+    registry_path: Path,
+    job_name_prefix: str = "eval",
+    default_task: str,
+    default_model: str = "anthropic/claude-haiku-4-5-20251001",
+) -> None:
+    """
+    Parse CLI args and run a Harbor job against tasks from this adapter.
+
+    Args:
+        handler_registry:  dict mapping dataset_name → DatasetHandler
+        tasks_dir:         local directory containing generated Harbor task dirs
+        registry_path:     path to this adapter's registry.json
+        job_name_prefix:   prefix for generated job names (e.g. "dsgym")
+        default_task:      default task or dataset to run (e.g. "daeval/0")
+        default_model:     default LLM model in provider/model format
+    """
+    parser = argparse.ArgumentParser(formatter_class=argparse.RawDescriptionHelpFormatter)
+    parser.add_argument(
+        "task", nargs="?", default=default_task, help="Task (dataset/id) or dataset name"
+    )
+    parser.add_argument(
+        "-m", "--model", default=default_model, help="LLM model (provider/model format)"
+    )
+    parser.add_argument("-k", "--n-attempts", default="1", help="Attempts per task (default: 1)")
+    parser.add_argument("-n", "--n-concurrent", default="4", help="Concurrent trials (default: 4)")
+    parser.add_argument("-l", "--n-tasks", help="Max tasks to run from dataset")
+    parser.add_argument("--max-turns", help="Max agent turns")
+    parser.add_argument("--force-build", action="store_true", help="Force Docker image rebuild")
+    parser.add_argument(
+        "--artifact",
+        action="append",
+        metavar="PATH",
+        help="Container path to capture as artifact (repeatable)",
+    )
+    args = parser.parse_args()
+
+    task_slug = args.task.replace("/", "__")
+    job_name = f"{job_name_prefix}__{task_slug}__{int(time.time()) % 100_000}"
+    harbor_bin = Path(sys.executable).parent / "harbor"
+
+    if _is_single_task(args.task):
+        task_path = tasks_dir / args.task
+        if not task_path.exists():
+            raise SystemExit(f"Task not found: {task_path}")
+        source_flags = ["-p", str(task_path)]
+    else:
+        if not registry_path.exists():
+            raise SystemExit(f"registry.json not found at {registry_path} — run 'just data' first")
+        source_flags = ["-d", args.task.lower(), "--registry-path", str(registry_path)]
+
+    cmd = [
+        str(harbor_bin),
+        "jobs",
+        "start",
+        *source_flags,
+        "-a",
+        "terminus-2",
+        "-m",
+        args.model,
+        "--n-attempts",
+        args.n_attempts,
+        "--n-concurrent",
+        args.n_concurrent,
+        "--job-name",
+        job_name,
+        "-y",
+    ]
+    if args.n_tasks:
+        cmd += ["--n-tasks", args.n_tasks]
+    if args.max_turns:
+        cmd += ["--ak", f"max_turns={args.max_turns}"]
+    if args.force_build:
+        cmd += ["--force-build"]
+
+    # Forward handler-declared artifacts and verifier env vars
+    dataset_name = args.task.split("/")[0]
+    handler = handler_registry.get(dataset_name)
+
+    artifacts = list(args.artifact or [])
+    if handler:
+        for path in handler.artifacts():
+            if path not in artifacts:
+                artifacts.append(path)
+    for artifact in artifacts:
+        cmd += ["--artifact", artifact]
+
+    if handler:
+        for key in handler.verifier_env_keys():
+            if val := os.environ.get(key):
+                cmd += ["--ve", f"{key}={val}"]
+
+    subprocess.run(cmd, check=True)

harborforge-1.0.0.dist-info/METADATA

@@ -0,0 +1,8 @@
+Metadata-Version: 2.4
+Name: harborforge
+Version: 1.0.0
+Requires-Python: ==3.13.*
+Provides-Extra: dev
+Requires-Dist: mypy>=1.10; extra == 'dev'
+Requires-Dist: pytest>=8.0; extra == 'dev'
+Requires-Dist: ruff>=0.8; extra == 'dev'

harborforge-1.0.0.dist-info/RECORD

@@ -0,0 +1,9 @@
+harborforge/__init__.py,sha256=QfgLX95O8k5nUGBfkZ_DEU5Kth74F_KCdK7-H2fSsMQ,216
+harborforge/mapper.py,sha256=fl8jQBsPvY_OJb6ZNoTZZ-I4vov9hRProS3EsIxnt0U,5588
+harborforge/registry.py,sha256=6VshQrjnMoQQjrUuwgj1Rg4DkJc_fchVyAHvDYvYDVM,1549
+harborforge/run.py,sha256=opKStzKmNvSCa6BSyMnGcQxJdLOLyUBaVzhZH6EjWc4,3959
+harborforge/handlers/__init__.py,sha256=14x5nOh5-Rt8J_yZpQCqeD0Xod0eLQmF3VZjVV5uFhA,363
+harborforge/handlers/base.py,sha256=yopagTLuo_k6CKfZuMcaeaUzk9rdTuayz-hIqj-MNyw,6591
+harborforge-1.0.0.dist-info/METADATA,sha256=1XhEcaG-22cVfMUuYXDXtZcPzHrrGTILf0gSGvrFX-c,227
+harborforge-1.0.0.dist-info/WHEEL,sha256=mffPy8wBnZQn2VnJUU5jE99KsxaSfiyMHV9Yt0aLVxs,87
+harborforge-1.0.0.dist-info/RECORD,,

harborforge-1.0.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.30.1
+Root-Is-Purelib: true
+Tag: py3-none-any