latch-eval-tools 0.1.0__py3-none-any.whl

latch_eval_tools/wrapper_entrypoint.py

@@ -0,0 +1,316 @@
+#!/usr/bin/env python3
+
+import asyncio
+import os
+import signal
+import sys
+from pathlib import Path
+
+
+def setup_environment(sandbox_dir: Path, notebook_id: str):
+    sandbox_dir.mkdir(parents=True, exist_ok=True)
+
+    root_dir = sandbox_dir / "root"
+    root_dir.mkdir(exist_ok=True)
+
+    latch_dir = root_dir / ".latch"
+    latch_dir.mkdir(exist_ok=True)
+
+    user_latch_token = Path.home() / ".latch" / "token"
+    if not user_latch_token.exists():
+        raise RuntimeError("Latch token required at ~/.latch/token")
+    token = user_latch_token.read_text().strip()
+
+    (latch_dir / "token").write_text(token)
+    (latch_dir / "id").write_text("99999")
+    (latch_dir / "notebook-id").write_text(notebook_id)
+    (latch_dir / "session-id").write_text("local-eval-session")
+    (latch_dir / "nucleus-url").write_text("https://nucleus.latch.bio")
+
+    os.environ.update({
+        "DD_VERSION": "local-dev",
+        "DD_SERVICE": "latch-plots-eval",
+        "DD_ENV": "local",
+        "DD_AGENT_HOST": "localhost",
+        "DD_TRACE_ENABLED": "false",
+        "DD_PROFILING_ENABLED": "false",
+        "DD_RUNTIME_METRICS_ENABLED": "false",
+        "OTEL_SDK_DISABLED": "true",
+        "auth_jwks_url": "https://example.com/jwks",
+        "auth_issuer": "local-dev",
+        "auth_audience": "local-dev",
+        "auth_self_signed_jwk": "{}",
+        "auto_reload": "false",
+        "logging_mode": "console",
+        "domain": "latch.bio",
+        "AGENT_DEBUG": "1",
+        "LATCH_SANDBOX_ROOT": str(latch_dir),
+        "LD_LIBRARY_PATH": f"/root/miniconda3/envs/plots-faas/lib:{os.environ.get('LD_LIBRARY_PATH', '')}",
+    })
+
+    import pathlib
+    original_path_new = pathlib.Path.__new__
+
+    def patched_path_new(cls, *args, **kwargs):
+        if args and str(args[0]) == "/root/.latch":
+            return original_path_new(cls, str(latch_dir), *args[1:], **kwargs)
+        return original_path_new(cls, *args, **kwargs)
+
+    pathlib.Path.__new__ = patched_path_new
+
+    venv_bin = str(Path(sys.executable).parent)
+    os.environ["PATH"] = f"{venv_bin}:{os.environ.get('PATH', '')}"
+
+    print(f"[wrapper] Using sandbox: {sandbox_dir}", flush=True)
+    print(f"[wrapper] Latch dir: {latch_dir}", flush=True)
+    print(f"[wrapper] Added {venv_bin} to PATH", flush=True)
+    return latch_dir
+
+
+async def mock_add_pod_event(*, auth, event_type):
+    print(f"[wrapper] Pod event (mocked): {event_type}", flush=True)
+
+
+async def run_server(latch_dir: Path, port: int, notebook_id: str):
+    from hypercorn.asyncio import serve
+    from hypercorn.config import Config as HypercornConfig
+    from latch_asgi.server import LatchASGIServer
+
+    from runtime.mount.endpoints import http_routes, websocket_routes
+    from runtime.mount.entrypoint import shutdown
+    from runtime.mount import entrypoint, headless_browser
+
+    print(f"[wrapper] Using real notebook {notebook_id}", flush=True)
+
+    entrypoint.add_pod_event = mock_add_pod_event
+
+    entrypoint.plots_ctx_manager.session_owner = "eval-harness"
+    original_screenshot = headless_browser.HeadlessBrowser.screenshot
+
+    async def patched_screenshot(self, path: str):
+        if path.startswith("/var/log/"):
+            path = str(latch_dir / path.split("/")[-1])
+        return await original_screenshot(self, path)
+
+    async def patched_start(self, notebook_url, local_storage, *, timeout_ms=30000):
+        from playwright.async_api import async_playwright
+        import json as json_mod
+        from collections.abc import Mapping
+
+        notebook_url = f"https://console.latch.bio/plots/{notebook_id}"
+        print(f"[wrapper] Patched notebook URL to: {notebook_url}", flush=True)
+
+        self.playwright = await async_playwright().start()
+        self.browser = await self.playwright.chromium.launch(headless=True)
+        self.page = await self.browser.new_page(viewport={"width": 1280, "height": 800})
+
+        storage = dict(local_storage) if isinstance(local_storage, Mapping) else local_storage
+        serialized = json_mod.dumps(storage)
+        await self.page.add_init_script(
+            f"""
+            const entries = JSON.parse({json_mod.dumps(serialized)});
+            for (const [k, v] of Object.entries(entries)) {{
+                localStorage.setItem(k, v);
+            }}
+            """
+        )
+
+        def should_log_browser_msg(msg):
+            text = msg.text
+            if "ERR_FAILED" in text or "[Network error]" in text:
+                return False
+            if "Failed to load resource" in text:
+                return False
+            return True
+
+        self.page.on("console", lambda msg: print(f"[browser] {msg.type}: {msg.text}", flush=True) if should_log_browser_msg(msg) else None)
+        self.page.on("pageerror", lambda err: print(f"[browser-error] Page error: {err}", flush=True))
+
+        async def handle_api_calls(route):
+            url = route.request.url
+
+            if "graphql" in url or "vacuole" in url:
+                try:
+                    post_data = route.request.post_data
+                    if post_data:
+                        import re
+                        query_match = re.search(r'(query|mutation)\s+(\w+)', post_data)
+                        query_name = query_match.group(2) if query_match else "unknown"
+                        print(f"[browser-gql] GraphQL request: {query_name}", flush=True)
+
+                        fake_pod = {
+                            "__typename": "PodInfo",
+                            "id": "99999",
+                            "status": "RUNNING",
+                            "jupyterToken": "eval-token",
+                            "cpuMillicores": "4000",
+                            "memoryBytes": "8589934592",
+                            "gpus": "0",
+                            "gpuType": None,
+                            "storageGigs": "50",
+                            "usedStorageGigs": "0",
+                            "internalIpAddress": "127.0.0.1",
+                            "autoShutoffDelay": {
+                                "__typename": "Interval",
+                                "days": 0, "months": 0, "hours": 1,
+                                "minutes": 0, "seconds": 0, "years": 0
+                            },
+                            "deployment": {
+                                "__typename": "PodDeployment",
+                                "id": "99999",
+                                "targetRegion": "us-west-2",
+                                "targetDomain": "us-west-2"
+                            }
+                        }
+
+                        pod_list_queries = ["GetPodStatus", "GetPodInfoFromNotebookId"]
+                        if query_name in pod_list_queries:
+                            print(f"[browser-gql] Intercepting {query_name} (podInfos), returning fake RUNNING pod", flush=True)
+                            fake_pod_response = json_mod.dumps({
+                                "data": {
+                                    "podInfos": {
+                                        "__typename": "PodInfosConnection",
+                                        "nodes": [fake_pod]
+                                    }
+                                }
+                            })
+                            await route.fulfill(status=200, content_type="application/json", body=fake_pod_response)
+                            return
+
+                        if query_name == "PodInfoByPodId":
+                            print(f"[browser-gql] Intercepting {query_name} (podInfo), returning fake RUNNING pod", flush=True)
+                            fake_pod_response = json_mod.dumps({
+                                "data": {
+                                    "podInfo": fake_pod
+                                }
+                            })
+                            await route.fulfill(status=200, content_type="application/json", body=fake_pod_response)
+                            return
+                except Exception as e:
+                    print(f"[browser-gql] Error checking GQL query: {e}", flush=True)
+                await route.continue_()
+            elif "nucleus" in url:
+                print(f"[browser-api] Blocking nucleus API call: {url}", flush=True)
+                await route.fulfill(status=200, content_type="application/json", body='{"data":{}}')
+            else:
+                await route.continue_()
+
+        await self.page.route("**/*", handle_api_calls)
+
+        print(f"[wrapper] Headless browser navigating to: {notebook_url}", flush=True)
+        await self.page.goto(notebook_url, wait_until="load")
+
+        try:
+            await self.page.wait_for_selector("[data-plot-ready='true']", timeout=timeout_ms)
+        except Exception:
+            await self.screenshot(str(latch_dir / "headless_browser_no_selector.png"))
+            raise
+
+        await self.screenshot(str(latch_dir / "headless_browser_ready.png"))
+        print("[wrapper] Headless browser page ready", flush=True)
+
+    headless_browser.HeadlessBrowser.screenshot = patched_screenshot
+    headless_browser.HeadlessBrowser.start = patched_start
+
+    async def patched_start_agent_proc():
+        conn_a = entrypoint.a_proc.conn_a = await entrypoint.SocketIo.from_socket(entrypoint.sock_a)
+
+        entrypoint.async_tasks.append(
+            asyncio.create_task(entrypoint.handle_agent_messages(conn_a))
+        )
+
+        print("[wrapper] Starting agent subprocess", flush=True)
+        entrypoint.a_proc.proc = await asyncio.create_subprocess_exec(
+            sys.executable,
+            "-u",
+            (entrypoint.dir_p / "agent.py"),
+            str(entrypoint.sock_agent_fd),
+            pass_fds=[entrypoint.sock_agent_fd],
+            stdin=asyncio.subprocess.DEVNULL,
+            stdout=asyncio.subprocess.PIPE,
+            stderr=asyncio.subprocess.PIPE,
+            env={
+                **os.environ,
+                "LATCH_SANDBOX_ROOT": str(latch_dir),
+                "PYTHONUNBUFFERED": "1",
+            },
+            preexec_fn=lambda: os.nice(5),
+        )
+
+        async def stream_output(stream, prefix=""):
+            while True:
+                line = await stream.readline()
+                if not line:
+                    break
+                print(f"[agent] {prefix}{line.decode().rstrip()}", flush=True)
+
+        asyncio.create_task(stream_output(entrypoint.a_proc.proc.stdout, ""))
+        asyncio.create_task(stream_output(entrypoint.a_proc.proc.stderr, "[stderr] "))
+        print(f"[wrapper] Agent subprocess started, PID: {entrypoint.a_proc.proc.pid}", flush=True)
+
+    entrypoint.start_agent_proc = patched_start_agent_proc
+
+    latch_server = LatchASGIServer(
+        http_routes=http_routes,
+        websocket_routes=websocket_routes,
+        startup_tasks=[entrypoint.start_kernel_proc(), patched_start_agent_proc()],
+        shutdown_tasks=[shutdown()],
+    )
+
+    cfg = HypercornConfig()
+    cfg.bind = [f"127.0.0.1:{port}"]
+    cfg.graceful_timeout = 0.1
+
+    print(f"\n[wrapper] Server starting on port {port}", flush=True)
+    print(f"[wrapper] WebSocket: ws://127.0.0.1:{port}/agent", flush=True)
+    print(f"[wrapper] HTTP: http://127.0.0.1:{port}/readyz", flush=True)
+
+    shutdown_event = asyncio.Event()
+
+    async def await_shutdown():
+        await shutdown_event.wait()
+
+    def shutdown_signal(*args):
+        print("\n[wrapper] Shutting down...", flush=True)
+        shutdown_event.set()
+
+    signal.signal(signal.SIGINT, shutdown_signal)
+    signal.signal(signal.SIGTERM, shutdown_signal)
+
+    try:
+        await serve(latch_server.raw_app, cfg, shutdown_trigger=await_shutdown)
+    finally:
+        await shutdown()
+
+
+def main():
+    import argparse
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--sandbox-dir", required=True, help="Sandbox directory path")
+    parser.add_argument("--port", type=int, default=5000, help="Server port")
+    parser.add_argument("--notebook-id", required=True, help="Notebook ID to use")
+    args = parser.parse_args()
+
+    sandbox_dir = Path(args.sandbox_dir)
+    
+    # Get the faas directory from environment or default location
+    faas_dir = Path(os.environ.get("LATCH_PLOTS_FAAS_PATH", "/root/latch-plots-faas"))
+    mount_dir = faas_dir / "runtime" / "mount"
+
+    sys.path.insert(0, str(faas_dir))
+    sys.path.insert(0, str(mount_dir))
+    sys.path.insert(0, str(mount_dir / "python_lib"))
+
+    latch_dir = setup_environment(sandbox_dir, args.notebook_id)
+
+    from latch_o11y.o11y import setup as setup_o11y
+    setup_o11y()
+
+    try:
+        asyncio.run(run_server(latch_dir, args.port, args.notebook_id))
+    except KeyboardInterrupt:
+        print("\n[wrapper] Stopped", flush=True)
+
+
+if __name__ == "__main__":
+    main()

latch_eval_tools-0.1.0.dist-info/METADATA

@@ -0,0 +1,118 @@
+Metadata-Version: 2.4
+Name: latch-eval-tools
+Version: 0.1.0
+Summary: Shared eval tools for single-cell bench, spatial bench, and future biology benchmarks.
+License-File: LICENSE
+Requires-Python: >=3.10
+Requires-Dist: aiohttp>=3.0.0
+Requires-Dist: anthropic>=0.72.0
+Requires-Dist: latch-config>=0.1.0
+Requires-Dist: latch>=2.0.0
+Requires-Dist: matplotlib>=3.0.0
+Requires-Dist: mini-swe-agent
+Requires-Dist: numpy>=1.24.0
+Requires-Dist: openai>=1.0.0
+Requires-Dist: orjson>=3.0.0
+Requires-Dist: pydantic>=2.0.0
+Requires-Dist: scikit-learn>=1.3.0
+Requires-Dist: scipy>=1.10.0
+Requires-Dist: statsmodels>=0.14.0
+Requires-Dist: websockets>=12.0
+Description-Content-Type: text/markdown
+
+# latch-eval-tools
+
+Shared eval tools for single-cell bench, spatial bench, and future biology benchmarks.
+
+## Installation
+
+```bash
+pip install latch-eval-tools
+```
+
+## Components
+
+### Types
+
+```python
+from latch_eval_tools import Eval, EvalResult
+
+eval_case = Eval(
+    id="test_001",
+    task="Count cells in the dataset",
+    data_node="latch:///data/sample.h5ad",
+    grader={"type": "numeric_tolerance", "config": {...}}
+)
+```
+
+### Graders
+
+Available graders: `numeric_tolerance`, `label_set_jaccard`, `distribution_comparison`, `marker_gene_precision_recall`, `marker_gene_separation`, `spatial_adjacency`, `multiple_choice`
+
+```python
+from latch_eval_tools.graders import get_grader, NumericToleranceGrader
+
+grader = get_grader("numeric_tolerance")
+result = grader.evaluate(
+    agent_answer={"n_cells": 1523},
+    config={
+        "ground_truth": {"n_cells": 1500},
+        "tolerances": {"n_cells": {"type": "relative", "value": 0.05}}
+    }
+)
+print(result.passed)
+print(result.reasoning)
+```
+
+### Harness
+
+Run evaluations with different agents:
+
+```python
+from latch_eval_tools.harness import EvalRunner, run_minisweagent_task
+
+runner = EvalRunner("evals/count_cells.json", cache_name=".scbench")
+result = runner.run(agent_function=lambda task, work_dir: 
+    run_minisweagent_task(task, work_dir, model_name="anthropic/claude-sonnet-4")
+)
+
+def my_agent(task_prompt: str, work_dir: Path) -> dict:
+    return {"answer": json.loads((work_dir / "eval_answer.json").read_text())}
+
+runner.run(agent_function=my_agent)
+```
+
+Built-in agents: `run_minisweagent_task`, `run_claudecode_task`, `run_plotsagent_task`
+
+### Linter
+
+Validate eval JSON files:
+
+```bash
+eval-lint evals/my_dataset/
+eval-lint evals/ --format json
+```
+
+```python
+from latch_eval_tools.linter import lint_eval, lint_directory
+
+result = lint_eval("evals/test.json")
+print(result.passed, result.issues)
+```
+
+## Eval JSON Schema
+
+```json
+{
+  "id": "unique_test_id",
+  "task": "Task description for the agent",
+  "data_node": "latch:///path/to/data.h5ad",
+  "grader": {
+    "type": "numeric_tolerance",
+    "config": {
+      "ground_truth": {"field": 42},
+      "tolerances": {"field": {"type": "absolute", "value": 1}}
+    }
+  }
+}
+```

latch_eval_tools-0.1.0.dist-info/RECORD

@@ -0,0 +1,33 @@
+latch_eval_tools/__init__.py,sha256=BykCd2JcFvB5Qb0voGuxQAkzE7xE_PxPnBdhHxgwetw,1590
+latch_eval_tools/answer_extraction.py,sha256=uApcsxAEPZX5o8DgQohM5AK7P0s6vnfgLKndVHDmiG0,1360
+latch_eval_tools/eval_server.py,sha256=iD0gRw0LeB-V3c04EMITti1sVoEwAv9xKcnCdy7WP7Y,21836
+latch_eval_tools/faas_utils.py,sha256=aJoyNuP5GKMioBmP0sreNgnQvXVZma2Y1c0COaCwlBw,561
+latch_eval_tools/headless_eval_server.py,sha256=UjkL7_ZzJzFprlbzi73uBvkKadXq7YkNYCq41xHeods,27461
+latch_eval_tools/types.py,sha256=IKYA6aHnAAKXrPEFVpWY4Q0vo02tn60MJSNXxmU3V38,789
+latch_eval_tools/wrapper_entrypoint.py,sha256=qWIjSFOP5m8FnAnlZhwSPZu7p0NLo3j-RJosX3JohdQ,12712
+latch_eval_tools/cli/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+latch_eval_tools/cli/eval_lint.py,sha256=G7R85-Gw_so2stk02zb4GpBMML3G4j_w2tLGTx_Nc74,5813
+latch_eval_tools/graders/__init__.py,sha256=lA9NbxB4Dv990zqtSLs8j27q544HKha9OWiceWFowvc,1427
+latch_eval_tools/graders/base.py,sha256=wRN47ilf0iXSE6cLF5gpQ91EaDHO6wepkCsgJ6vBkpc,770
+latch_eval_tools/graders/distribution.py,sha256=7u9pwPC45ClGTmEvnYPGe3pTNrgkCFUeP412v3-icps,4266
+latch_eval_tools/graders/label_set.py,sha256=CXhLZ4qdOvRMYgVxh3x0m45r7IyXo8nJfrbiNuEfm0Y,2767
+latch_eval_tools/graders/marker_gene.py,sha256=ysFPXx_xZ9rhusvPgcVIpcA7-kU_wSTFR9r9dSZjHi0,13486
+latch_eval_tools/graders/multiple_choice.py,sha256=HTyYWQGFv7ATmTNJMhUnisiuNb1F7VVx4nU-pJ0KV0o,1412
+latch_eval_tools/graders/numeric.py,sha256=Q19GtyYtatKRdPX9DlWJ8CKSQHUhJ6P6k7hnhZnKE04,6826
+latch_eval_tools/graders/spatial.py,sha256=8dNRsCVazCI98mHwhLHz0T-kyvLL4dTrsayu2DqjMdU,4258
+latch_eval_tools/harness/__init__.py,sha256=FyDDHRdEOUo845NIGqzF1CbOzaZfilsId8LxhtgEWHQ,764
+latch_eval_tools/harness/claudecode.py,sha256=gbbPtie4TALwkQ7zas5rPNtkL-SjA4yDVeDfoX1hWuY,7023
+latch_eval_tools/harness/minisweagent.py,sha256=SZwbnigvGOWU4tIgijJSyOr4urTe8n1Dgj6ViuL2uM4,9197
+latch_eval_tools/harness/plotsagent.py,sha256=Qw_q-6WzAsGo1RzKBzD0lzsXZeyGRGwfZ0AU5s_e00g,5017
+latch_eval_tools/harness/runner.py,sha256=f042LLhEAN4rvltPIhjSn588Z2AkP5tTk0pKDH69i2M,7264
+latch_eval_tools/harness/utils.py,sha256=__fy4_eJqEzU7cMqd8fcztwfxKH0jvWCYyUCyHvdhdQ,5880
+latch_eval_tools/linter/__init__.py,sha256=fKRgnVvL_eJduYFagRJrzE4RI5kI9G6VsiyI0dRnc3k,541
+latch_eval_tools/linter/explanations.py,sha256=Uzg-IH9YjZl__X7nF7lDc-4FSv619CE8LyXo8PkO0tk,14308
+latch_eval_tools/linter/runner.py,sha256=zii4EbfsdWqdynzCxn1GLX0oC7rUgnrq0Vwsc-13YNk,4619
+latch_eval_tools/linter/schema.py,sha256=Acqs4VSwhOP4G85nUCghPquGMGQ413In7kAWaQ1BtNE,3688
+latch_eval_tools/linter/validators.py,sha256=iWcWBrcOmjv8abJEy0ch6Di2LbT5L8TVTe2cl88mQoU,21488
+latch_eval_tools-0.1.0.dist-info/METADATA,sha256=qKASKwb8G3-KSMKJHemNt_VbhF6LsoPU4wOUXhZjCXY,2887
+latch_eval_tools-0.1.0.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
+latch_eval_tools-0.1.0.dist-info/entry_points.txt,sha256=pxTKTp2rBe7xq_j1KSVZE6ULxDZv_BsbFjAGHEw7Tlo,66
+latch_eval_tools-0.1.0.dist-info/licenses/LICENSE,sha256=zepfPTPE3p7B8XbkQVZ-MeZwB9vmSoXQhMUvHNnlvQo,39
+latch_eval_tools-0.1.0.dist-info/RECORD,,

latch_eval_tools-0.1.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.28.0
+Root-Is-Purelib: true
+Tag: py3-none-any

latch_eval_tools-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+eval-lint = latch_eval_tools.cli.eval_lint:main

latch_eval_tools-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1 @@
+© LatchBio LLC. All rights reserved.