hte-cli 0.1.27__tar.gz → 0.1.28__tar.gz

{hte_cli-0.1.27 → hte_cli-0.1.28}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: hte-cli
-Version: 0.1.27
+Version: 0.1.28
 Summary: Human Time-to-Completion Evaluation CLI
 Project-URL: Homepage, https://github.com/sean-peters-au/lyptus-mono
 Author: Lyptus Research

{hte_cli-0.1.27 → hte_cli-0.1.28}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "hte-cli"
-version = "0.1.27"
+version = "0.1.28"
 description = "Human Time-to-Completion Evaluation CLI"
 readme = "README.md"
 requires-python = ">=3.11"

{hte_cli-0.1.27 → hte_cli-0.1.28}/src/hte_cli/cli.py

@@ -397,10 +397,16 @@ def tasks_run(ctx, task_id: str | None):
     # Step 5: Pre-pull Docker images with progress
     from hte_cli.image_utils import extract_images_from_compose
     import re
+    import time
+
+    setup_start_time = time.monotonic()
+    images: list[str] = []
+    results: list[tuple[str, bool, str]] = []
 
     if compose_yaml:
         images = extract_images_from_compose(compose_yaml)
         if images:
+            events.setup_started(images)
             console.print()
             console.print(f"[bold]Preparing Docker environment ({len(images)} images)...[/bold]")
 
@@ -533,14 +539,27 @@ def tasks_run(ctx, task_id: str | None):
                     console.print(f"  [red]✗[/red] {short_name} [dim](failed)[/dim]")
                     results.append((img, False, "failed"))
 
-            failed = sum(1 for _, ok, _ in results if not ok)
-            if failed > 0:
+            failed_count = sum(1 for _, ok, _ in results if not ok)
+            if failed_count > 0:
                 console.print(
-                    f"[yellow]Warning: {failed} image(s) failed to pull. "
+                    f"[yellow]Warning: {failed_count} image(s) failed to pull. "
                     "Task may fail to start.[/yellow]"
                 )
             console.print()
 
+    # Record image pull timing
+    if images:
+        pull_duration = time.monotonic() - setup_start_time
+        pulled = [img for img, ok, status in results if ok and status == "pulled"]
+        cached = [img for img, ok, status in results if ok and status == "cached"]
+        failed = [img for img, ok, status in results if not ok]
+        events.image_pull_completed(
+            duration_seconds=pull_duration,
+            pulled=pulled,
+            cached=cached,
+            failed=failed,
+        )
+
     # Step 6: Run Inspect's human_cli
     runner = TaskRunner()
     console.print("[bold]Starting task environment...[/bold]")
@@ -549,6 +568,10 @@ def tasks_run(ctx, task_id: str | None):
 
     events.docker_started()
 
+    # Record total setup time (image pulls + compose up)
+    total_setup = time.monotonic() - setup_start_time
+    events.setup_completed(total_seconds=total_setup)
+
     eval_log_bytes = None
     local_eval_path = None
     try:
@@ -612,13 +635,14 @@ def tasks_run(ctx, task_id: str | None):
         except Exception:
             pass  # Not a CyberGym task or malformed zip
 
-    # Show upload size info
-    upload_size_kb = 0
-    if eval_log_bytes:
-        upload_size_kb = len(eval_log_bytes) / 1024
-        size_mb = upload_size_kb / 1024
-        if size_mb > 50:
-            console.print(f"[yellow]Warning: Large eval log ({size_mb:.1f} MB)[/yellow]")
+    # Show upload size info and track timing
+    upload_size_bytes = len(eval_log_bytes) if eval_log_bytes else 0
+    upload_size_kb = upload_size_bytes / 1024
+    if upload_size_kb / 1024 > 50:
+        console.print(f"[yellow]Warning: Large eval log ({upload_size_kb / 1024:.1f} MB)[/yellow]")
+
+    events.upload_started(size_bytes=upload_size_bytes)
+    upload_start_time = time.monotonic()
 
     with Progress(
         SpinnerColumn(),
@@ -645,6 +669,10 @@ def tasks_run(ctx, task_id: str | None):
             console.print("[yellow]Your result was saved locally but not uploaded.[/yellow]")
             sys.exit(1)
 
+    # Record upload completion
+    upload_duration = time.monotonic() - upload_start_time
+    events.upload_completed(duration_seconds=upload_duration, size_bytes=upload_size_bytes)
+
     console.print()
     console.print("[green]Result uploaded successfully![/green]")
 

{hte_cli-0.1.27 → hte_cli-0.1.28}/src/hte_cli/events.py

@@ -24,6 +24,12 @@ class EventStreamer:
         "docker_started",
         "docker_stopped",
         "session_completed",
+        # Overhead tracking events
+        "setup_started",
+        "image_pull_completed",
+        "setup_completed",
+        "upload_started",
+        "upload_completed",
     }
 
     def __init__(self, api: APIClient, session_id: str):
@@ -126,3 +132,45 @@ class EventStreamer:
         if answer is not None:
             data["answer_submitted"] = True
         return self.send("session_completed", data or None)
+
+    # Overhead tracking events
+
+    def setup_started(self, images: list[str]) -> bool:
+        """Record start of setup phase (before image pulls)."""
+        return self.send("setup_started", {"images": images})
+
+    def image_pull_completed(
+        self,
+        duration_seconds: float,
+        pulled: list[str],
+        cached: list[str],
+        failed: list[str],
+    ) -> bool:
+        """Record image pull results with timing."""
+        return self.send(
+            "image_pull_completed",
+            {
+                "duration_seconds": duration_seconds,
+                "pulled": pulled,
+                "cached": cached,
+                "failed": failed,
+            },
+        )
+
+    def setup_completed(self, total_seconds: float) -> bool:
+        """Record end of setup phase (environment ready for work)."""
+        return self.send("setup_completed", {"total_seconds": total_seconds})
+
+    def upload_started(self, size_bytes: int) -> bool:
+        """Record start of result upload."""
+        return self.send("upload_started", {"size_bytes": size_bytes})
+
+    def upload_completed(self, duration_seconds: float, size_bytes: int) -> bool:
+        """Record end of result upload with timing."""
+        return self.send(
+            "upload_completed",
+            {
+                "duration_seconds": duration_seconds,
+                "size_bytes": size_bytes,
+            },
+        )