hte-cli 0.1.27__py3-none-any.whl → 0.2.0__py3-none-any.whl

hte_cli/api_client.py

@@ -201,6 +201,31 @@ class APIClient:
             },
         )
 
+    # =========================================================================
+    # Session-based API (new flow: hte-cli session join <session_id>)
+    # =========================================================================
+
+    def join_session(self, session_id: str) -> dict:
+        """Join an existing session created by web UI.
+
+        Returns session info including task data, benchmark, mode, etc.
+        Sets cli_connected_at on the server.
+        """
+        return self.post(f"/sessions/{session_id}/join")
+
+    def get_session_files(self, session_id: str) -> bytes:
+        """Download task files for a session as zip."""
+        return self.get_raw(f"/sessions/{session_id}/files")
+
+    def get_session_compose(self, session_id: str) -> str:
+        """Get compose.yaml content for a session."""
+        content = self.get_raw(f"/sessions/{session_id}/compose")
+        return content.decode("utf-8")
+
+    # =========================================================================
+    # Result Upload
+    # =========================================================================
+
     def upload_result(
         self,
         session_id: str,

hte_cli/cli.py

@@ -147,13 +147,239 @@ def auth_status(ctx):
 
 
 # =============================================================================
-# Tasks Commands
+# Session Commands (New flow: session join <session_id>)
+# =============================================================================
+
+
+@cli.group()
+def session():
+    """Session management commands."""
+    pass
+
+
+@session.command("join")
+@click.argument("session_id")
+@click.option("--force-setup", is_flag=True, help="Re-run setup even if reconnecting")
+@click.pass_context
+def session_join(ctx, session_id: str, force_setup: bool):
+    """Join an existing session by ID.
+
+    This is the primary way to start working on a task:
+    1. Start the task from the web UI (creates session)
+    2. Run this command with the session ID shown in the web UI
+    3. The environment will be set up and the timer will start
+    """
+    config: Config = ctx.obj["config"]
+
+    if not config.is_authenticated():
+        console.print("[red]Not logged in. Run: hte-cli auth login[/red]")
+        sys.exit(1)
+
+    api = APIClient(config)
+
+    # Step 1: Join session
+    console.print()
+    with Progress(
+        SpinnerColumn(),
+        TextColumn("[progress.description]{task.description}"),
+        console=console,
+    ) as progress:
+        progress.add_task("Joining session...", total=None)
+
+        try:
+            session_info = api.join_session(session_id)
+        except APIError as e:
+            if "Invalid session ID format" in str(e):
+                console.print(f"[red]{e}[/red]")
+            elif e.status_code == 404:
+                console.print("[red]Session not found. Check the session ID and try again.[/red]")
+            elif e.status_code == 400 and "paused" in str(e).lower():
+                console.print("[yellow]Session is paused. Please resume from the web UI first.[/yellow]")
+            else:
+                console.print(f"[red]Error: {e}[/red]")
+            sys.exit(1)
+
+    # Check if reconnecting (session already in_progress)
+    is_reconnect = session_info.get("status") == "in_progress"
+
+    if is_reconnect and not force_setup:
+        console.print("[yellow]Reconnecting to existing session...[/yellow]")
+        console.print()
+
+    console.print(
+        Panel(
+            f"[bold]Task:[/bold] {session_info['task_id']}\n"
+            f"[bold]Benchmark:[/bold] {session_info['benchmark']}\n"
+            f"[bold]Mode:[/bold] {session_info['mode']}",
+            title="Session Joined",
+        )
+    )
+    console.print()
+
+    # Import runner components
+    from hte_cli.events import EventStreamer
+    from hte_cli.runner import TaskRunner, DockerComposeManager
+
+    # Create event streamer
+    events = EventStreamer(api, session_id)
+
+    # Step 2: Download task files and compose (skip if reconnecting without force)
+    files_zip = None
+    compose_yaml = None
+
+    if not is_reconnect or force_setup:
+        with Progress(
+            SpinnerColumn(),
+            TextColumn("[progress.description]{task.description}"),
+            console=console,
+        ) as progress:
+            task_id_display = progress.add_task("Downloading task files...", total=None)
+            try:
+                files_zip = api.get_session_files(session_id)
+            except APIError as e:
+                # Files are optional for some benchmarks
+                console.print(f"[dim]Note: {e}[/dim]")
+                files_zip = None
+
+            progress.update(task_id_display, description="Downloading compose file...")
+            try:
+                compose_yaml = api.get_session_compose(session_id)
+            except APIError as e:
+                # Compose is optional for simple benchmarks
+                compose_yaml = None
+
+    # Validate compose for sandbox-required benchmarks
+    benchmark = session_info.get("benchmark", "").lower()
+    SANDBOX_REQUIRED_BENCHMARKS = {"cybench", "intercode-ctf", "cvebench", "cybergym", "nyuctf"}
+    if benchmark in SANDBOX_REQUIRED_BENCHMARKS and not compose_yaml and not is_reconnect:
+        console.print(
+            f"[red]Error: {benchmark} requires a Docker sandbox but no compose file was found.[/red]"
+        )
+        console.print()
+        console.print(
+            f"Please contact support: {SUPPORT_EMAIL}"
+        )
+        sys.exit(1)
+
+    # Build assignment dict for runner compatibility
+    assignment = {
+        "assignment_id": session_info.get("assignment_id"),
+        "task_id": session_info["task_id"],
+        "benchmark": session_info["benchmark"],
+        "mode": session_info["mode"],
+        "time_cap_seconds": session_info.get("time_cap_seconds"),
+    }
+
+    # Build task dict for runner
+    task_data = {
+        "instructions": session_info.get("instructions", ""),
+        "metadata": session_info.get("metadata", {}),
+        "scorer_type": session_info.get("scorer_type"),
+        "intermediate_scoring": session_info.get("intermediate_scoring", False),
+    }
+
+    # Step 3: Run setup (skip if reconnecting without force)
+    if not is_reconnect or force_setup:
+        # Send setup_started event
+        events.setup_started({"cli_version": __version__, "task_id": assignment["task_id"]})
+
+        # Extract files and run compose
+        runner = TaskRunner(session_id, api, events, console)
+        compose_manager = None
+
+        if compose_yaml:
+            compose_manager = DockerComposeManager(
+                compose_yaml=compose_yaml,
+                files_zip=files_zip,
+                session_id=session_id,
+                task_id=assignment["task_id"],
+                console=console,
+            )
+
+            # Pull images
+            events.image_pull_started({})
+            compose_manager.pull_images()
+            events.image_pull_completed({})
+
+            # Start containers
+            compose_manager.up()
+
+        # Send setup_completed - THIS STARTS THE TIMER ON SERVER
+        events.setup_completed({"cli_version": __version__})
+        console.print("[green]Environment ready! Timer started.[/green]")
+        console.print()
+    else:
+        # Reconnecting - compose should already be running
+        console.print("[dim]Skipping setup (use --force-setup to re-run)[/dim]")
+        console.print()
+
+    # Step 4: Show instructions
+    if session_info.get("instructions"):
+        console.print(Panel(session_info["instructions"], title="Task Instructions"))
+        console.print()
+
+    # Step 5: Run the task interaction loop
+    runner = TaskRunner(session_id, api, events, console)
+    try:
+        result = runner.run(
+            assignment=assignment,
+            task=task_data,
+            compose_yaml=compose_yaml,
+            files_zip=files_zip,
+        )
+    except KeyboardInterrupt:
+        console.print()
+        console.print("[yellow]Interrupted. Session remains active - you can reconnect later.[/yellow]")
+        sys.exit(0)
+
+    # Step 6: Upload result
+    if result and result.answer:
+        events.session_completed(
+            elapsed_seconds=result.time_seconds,
+            answer=result.answer,
+        )
+
+        console.print()
+        console.print("[green]Task completed![/green]")
+        console.print(f"Answer: {result.answer}")
+        console.print(f"Time: {result.time_seconds:.1f}s")
+
+        # Upload to server
+        with Progress(
+            SpinnerColumn(),
+            TextColumn("[progress.description]{task.description}"),
+            console=console,
+        ) as progress:
+            progress.add_task("Uploading result...", total=None)
+            try:
+                upload_result = api.upload_result(
+                    session_id=session_id,
+                    answer=result.answer or "",
+                    client_active_seconds=result.time_seconds,
+                    eval_log_bytes=result.eval_log_bytes,
+                    score=result.score,
+                    score_binarized=result.score_binarized,
+                    agent_id=result.agent_id,
+                )
+            except APIError as e:
+                console.print(f"[red]Failed to upload result: {e}[/red]")
+                sys.exit(1)
+
+        if upload_result.get("score") is not None:
+            console.print(f"Score: {upload_result['score']}")
+
+        console.print()
+        console.print("[green]Done! Return to the web UI to see your results.[/green]")
+
+
+# =============================================================================
+# Tasks Commands (DEPRECATED - use 'session join' instead)
 # =============================================================================
 
 
 @cli.group()
 def tasks():
-    """Task commands."""
+    """Task commands (deprecated - use 'session join' instead)."""
     pass
 
 
@@ -223,7 +449,23 @@ def tasks_list(ctx):
 @click.argument("task_id", required=False)
 @click.pass_context
 def tasks_run(ctx, task_id: str | None):
-    """Run a task (default: highest priority pending task)."""
+    """[DEPRECATED] Run a task - use 'session join' instead."""
+    console.print()
+    console.print("[red]This command is deprecated.[/red]")
+    console.print()
+    console.print("The new workflow is:")
+    console.print("  1. Start the task from the web UI: https://cyber-task-horizons.com")
+    console.print("  2. Run the command shown: [bold]hte-cli session join <session_id>[/bold]")
+    console.print()
+    console.print("This ensures accurate timing by starting the timer only when")
+    console.print("the environment is ready, not including Docker setup time.")
+    console.print()
+    sys.exit(1)
+
+
+# Keep the old implementation as _tasks_run_legacy for testing if needed
+def _tasks_run_legacy(ctx, task_id: str | None):
+    """Legacy implementation of tasks run (for testing only)."""
     config: Config = ctx.obj["config"]
 
     if not config.is_authenticated():
@@ -397,10 +639,16 @@ def tasks_run(ctx, task_id: str | None):
     # Step 5: Pre-pull Docker images with progress
     from hte_cli.image_utils import extract_images_from_compose
     import re
+    import time
+
+    setup_start_time = time.monotonic()
+    images: list[str] = []
+    results: list[tuple[str, bool, str]] = []
 
     if compose_yaml:
         images = extract_images_from_compose(compose_yaml)
         if images:
+            events.setup_started(images)
             console.print()
             console.print(f"[bold]Preparing Docker environment ({len(images)} images)...[/bold]")
 
@@ -533,14 +781,27 @@ def tasks_run(ctx, task_id: str | None):
                     console.print(f"  [red]✗[/red] {short_name} [dim](failed)[/dim]")
                     results.append((img, False, "failed"))
 
-            failed = sum(1 for _, ok, _ in results if not ok)
-            if failed > 0:
+            failed_count = sum(1 for _, ok, _ in results if not ok)
+            if failed_count > 0:
                 console.print(
-                    f"[yellow]Warning: {failed} image(s) failed to pull. "
+                    f"[yellow]Warning: {failed_count} image(s) failed to pull. "
                     "Task may fail to start.[/yellow]"
                 )
             console.print()
 
+    # Record image pull timing
+    if images:
+        pull_duration = time.monotonic() - setup_start_time
+        pulled = [img for img, ok, status in results if ok and status == "pulled"]
+        cached = [img for img, ok, status in results if ok and status == "cached"]
+        failed = [img for img, ok, status in results if not ok]
+        events.image_pull_completed(
+            duration_seconds=pull_duration,
+            pulled=pulled,
+            cached=cached,
+            failed=failed,
+        )
+
     # Step 6: Run Inspect's human_cli
     runner = TaskRunner()
     console.print("[bold]Starting task environment...[/bold]")
@@ -549,6 +810,10 @@ def tasks_run(ctx, task_id: str | None):
 
     events.docker_started()
 
+    # Record total setup time (image pulls + compose up)
+    total_setup = time.monotonic() - setup_start_time
+    events.setup_completed(total_seconds=total_setup)
+
     eval_log_bytes = None
     local_eval_path = None
     try:
@@ -612,13 +877,14 @@ def tasks_run(ctx, task_id: str | None):
         except Exception:
             pass  # Not a CyberGym task or malformed zip
 
-    # Show upload size info
-    upload_size_kb = 0
-    if eval_log_bytes:
-        upload_size_kb = len(eval_log_bytes) / 1024
-        size_mb = upload_size_kb / 1024
-        if size_mb > 50:
-            console.print(f"[yellow]Warning: Large eval log ({size_mb:.1f} MB)[/yellow]")
+    # Show upload size info and track timing
+    upload_size_bytes = len(eval_log_bytes) if eval_log_bytes else 0
+    upload_size_kb = upload_size_bytes / 1024
+    if upload_size_kb / 1024 > 50:
+        console.print(f"[yellow]Warning: Large eval log ({upload_size_kb / 1024:.1f} MB)[/yellow]")
+
+    events.upload_started(size_bytes=upload_size_bytes)
+    upload_start_time = time.monotonic()
 
     with Progress(
         SpinnerColumn(),
@@ -645,6 +911,10 @@ def tasks_run(ctx, task_id: str | None):
             console.print("[yellow]Your result was saved locally but not uploaded.[/yellow]")
             sys.exit(1)
 
+    # Record upload completion
+    upload_duration = time.monotonic() - upload_start_time
+    events.upload_completed(duration_seconds=upload_duration, size_bytes=upload_size_bytes)
+
     console.print()
     console.print("[green]Result uploaded successfully![/green]")
 

hte_cli/events.py

@@ -24,6 +24,12 @@ class EventStreamer:
         "docker_started",
         "docker_stopped",
         "session_completed",
+        # Overhead tracking events
+        "setup_started",
+        "image_pull_completed",
+        "setup_completed",
+        "upload_started",
+        "upload_completed",
     }
 
     def __init__(self, api: APIClient, session_id: str):
@@ -126,3 +132,45 @@ class EventStreamer:
         if answer is not None:
             data["answer_submitted"] = True
         return self.send("session_completed", data or None)
+
+    # Overhead tracking events
+
+    def setup_started(self, images: list[str]) -> bool:
+        """Record start of setup phase (before image pulls)."""
+        return self.send("setup_started", {"images": images})
+
+    def image_pull_completed(
+        self,
+        duration_seconds: float,
+        pulled: list[str],
+        cached: list[str],
+        failed: list[str],
+    ) -> bool:
+        """Record image pull results with timing."""
+        return self.send(
+            "image_pull_completed",
+            {
+                "duration_seconds": duration_seconds,
+                "pulled": pulled,
+                "cached": cached,
+                "failed": failed,
+            },
+        )
+
+    def setup_completed(self, total_seconds: float) -> bool:
+        """Record end of setup phase (environment ready for work)."""
+        return self.send("setup_completed", {"total_seconds": total_seconds})
+
+    def upload_started(self, size_bytes: int) -> bool:
+        """Record start of result upload."""
+        return self.send("upload_started", {"size_bytes": size_bytes})
+
+    def upload_completed(self, duration_seconds: float, size_bytes: int) -> bool:
+        """Record end of result upload with timing."""
+        return self.send(
+            "upload_completed",
+            {
+                "duration_seconds": duration_seconds,
+                "size_bytes": size_bytes,
+            },
+        )

{hte_cli-0.1.27.dist-info → hte_cli-0.2.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: hte-cli
-Version: 0.1.27
+Version: 0.2.0
 Summary: Human Time-to-Completion Evaluation CLI
 Project-URL: Homepage, https://github.com/sean-peters-au/lyptus-mono
 Author: Lyptus Research
@@ -23,6 +23,10 @@ Requires-Dist: platformdirs>=4.0
 Requires-Dist: pydantic>=2.0
 Requires-Dist: pyyaml>=6.0
 Requires-Dist: rich>=13.0
+Provides-Extra: dev
+Requires-Dist: pexpect>=4.8; extra == 'dev'
+Requires-Dist: pytest>=7.0; extra == 'dev'
+Requires-Dist: requests>=2.28; extra == 'dev'
 Description-Content-Type: text/markdown
 
 # hte-cli

{hte_cli-0.1.27.dist-info → hte_cli-0.2.0.dist-info}/RECORD

@@ -1,15 +1,15 @@
 hte_cli/__init__.py,sha256=fDGXp-r8bIoLtlQnn5xJ_CpwMhonvk9bGjZQsjA2mDI,914
 hte_cli/__main__.py,sha256=63n0gNGfskidWDU0aAIF2N8lylVCLYKVIkrN9QiORoo,107
-hte_cli/api_client.py,sha256=mO4buDND5cIWESg4gSKb8WkdA1iPwkmTa0L3xL6lvNQ,8153
-hte_cli/cli.py,sha256=AolSWSqzeaAJKgsixPMqlwqfB24IGB60jhaDjdktF8A,28169
+hte_cli/api_client.py,sha256=m42kfFZS72Nu_VuDwxRsLNy4ziCcvgk7KNWBh9gwqy0,9257
+hte_cli/cli.py,sha256=mMoy1xHKwbeSEHh2CvfdeHJKle_HT4PwKursXBUc__c,38389
 hte_cli/config.py,sha256=42Xv__YMSeRLs2zhGukJkIXFKtnBtYCHnONfViGyt2g,3387
 hte_cli/errors.py,sha256=1J5PpxcUKBu6XjigMMCPOq4Zc12tnv8LhAsiaVFWLQM,2762
-hte_cli/events.py,sha256=LCNLPJuk_Sz-rCl1Aa3k28y10_jwAx3urbnz3OXYPmE,3937
+hte_cli/events.py,sha256=Zn-mroqaLHNzdT4DFf8st1Qclglshihdc09dBfCN070,5522
 hte_cli/image_utils.py,sha256=454yoZEI1duNYrZC8UjhfZzDRP4Nxdrf2TvnZ_54G1k,4439
 hte_cli/runner.py,sha256=DhC8FMjHwfLR193iP4thLDRZrNssYA9KH1WYKU2JKeg,13535
 hte_cli/scorers.py,sha256=sFoPJePRt-K191-Ga4cVmrldruJclYXTOLkU_C9nCDI,6025
 hte_cli/version_check.py,sha256=WVZyGy2XfAghQYdd2N9-0Qfg-7pgp9gt4761-PnmacI,1708
-hte_cli-0.1.27.dist-info/METADATA,sha256=Puk8nyCfgXXVtyt89PpQtKn4e3iQsATSRxon6FWw4E0,3615
-hte_cli-0.1.27.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
-hte_cli-0.1.27.dist-info/entry_points.txt,sha256=XbyEEi1H14DFAt0Kdl22e_IRVEGzimSzYSh5HlhKlFA,41
-hte_cli-0.1.27.dist-info/RECORD,,
+hte_cli-0.2.0.dist-info/METADATA,sha256=A2RhmrFnjGkpnbVwjel_CNTtL2Mg_LefwNeWejIvul8,3767
+hte_cli-0.2.0.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
+hte_cli-0.2.0.dist-info/entry_points.txt,sha256=XbyEEi1H14DFAt0Kdl22e_IRVEGzimSzYSh5HlhKlFA,41
+hte_cli-0.2.0.dist-info/RECORD,,