hud-python 0.2.7__tar.gz → 0.2.9__tar.gz

{hud_python-0.2.7 → hud_python-0.2.9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: hud-python
-Version: 0.2.7
+Version: 0.2.9
 Summary: SDK for the HUD evaluation platform.
 Project-URL: Homepage, https://github.com/hud-evals/hud-sdk
 Project-URL: Bug Tracker, https://github.com/hud-evals/hud-sdk/issues

{hud_python-0.2.7 → hud_python-0.2.9}/examples/sheetbench_direct_example.ipynb

@@ -14,6 +14,7 @@
     "export HUD_API_KEY=your_api_key_here\n",
     "pip install hud-python\n",
     "pip install pandas # optional for debugging\n",
+    "pip install openpyxl # also a requirement for pandas\n",
     "```"
    ]
   },
@@ -26,7 +27,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 70,
+   "execution_count": 2,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -51,24 +52,9 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 72,
+   "execution_count": null,
    "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Prompt: Sheet REV_QTR lists quarterly revenue from Q1-2022 through Q1-2025. Compute the compound annual growth rate between those two points. Enter the result in ANSWER!A1.\n",
-      "['REV_QTR']\n",
-      "   Quarter  Revenue\n",
-      "0  Q1-2022  4200000\n",
-      "1  Q2-2022  4404653\n",
-      "2  Q3-2022  4746171\n",
-      "3  Q4-2022  5062265\n",
-      "4  Q1-2023  5365661\n"
-     ]
-    }
-   ],
+   "outputs": [],
    "source": [
     "# Load in the first task for testing\n",
     "test_task = taskset[1]\n",
@@ -93,20 +79,9 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 73,
+   "execution_count": null,
    "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "['REV_QTR', 'ANSWER']\n",
-      "Empty DataFrame\n",
-      "Columns: [0.241955862]\n",
-      "Index: []\n"
-     ]
-    }
-   ],
+   "outputs": [],
    "source": [
     "###\n",
     "### Your agent loop goes here, using *prompt* and *input_xlsx_file*, returns *output_xlsx_file*\n",
@@ -129,15 +104,7 @@
    "cell_type": "code",
    "execution_count": null,
    "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Reward: 1.0\n"
-     ]
-    }
-   ],
+   "outputs": [],
    "source": [
     "# Get the base64 encoded xlsx file\n",
     "base64_output_xlsx_file = base64.b64encode(output_xlsx_file).decode(\"utf-8\")\n",
@@ -174,7 +141,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 64,
+   "execution_count": 3,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -185,11 +152,11 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 69,
+   "execution_count": 17,
    "metadata": {},
    "outputs": [],
    "source": [
-    "async def run_single_task(task):\n",
+    "async def run_single_task(task, job=None):\n",
     "    prompt = task.prompt\n",
     "    input_xlsx_file = requests.get(task.setup.args[0]).content\n",
     "\n",
@@ -202,7 +169,7 @@
     "    # Run evaluation\n",
     "    task.setup = (\"sheets_from_bytes\", base64_output_xlsx_file)\n",
     "    task.id = None\n",
-    "    env = await gym.make(task)\n",
+    "    env = await gym.make(task, job=job)\n",
     "    result = await env.evaluate()\n",
     "    await env.close()\n",
     "\n",
@@ -211,25 +178,22 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 68,
+   "execution_count": null,
    "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Average reward: 0.0\n"
-     ]
-    }
-   ],
+   "outputs": [],
    "source": [
     "# Loading and evaluating 50 tasks should take around 2 minutes, without the agent loop\n",
     "import asyncio\n",
     "\n",
+    "# Adds the job to the app.hud.so platform, optional\n",
+    "from hud import create_job\n",
+    "\n",
+    "# Run the taskset\n",
     "taskset = await load_taskset(\"SheetBench-50\", metadata={\"partial\": True})\n",
+    "job = await create_job(\"SheetBench-50-Excel-Agent\", evalset_id=taskset.id)\n",
     "\n",
-    "task_jobs = [run_single_task(task) for task in taskset]\n",
-    "rewards = await asyncio.gather(*task_jobs)\n",
+    "task_runs = [run_single_task(task, job) for task in taskset]\n",
+    "rewards = await asyncio.gather(*task_runs)\n",
     "\n",
     "print(f\"Average reward: {sum(rewards) / len(rewards)}\")"
    ]