fleet-python 0.2.79__py3-none-any.whl → 0.2.80__py3-none-any.whl

examples/iterate_verifiers.py

@@ -5,6 +5,14 @@ import sys
 from typing import Dict, Tuple, Optional
 
 
+# Marker for storing leading content (docstrings, imports) in the Python file
+LEADING_CONTENT_START = "# @LEADING_CONTENT_START"
+LEADING_CONTENT_END = "# @LEADING_CONTENT_END"
+# Legacy markers for backwards compatibility
+LEADING_DOCSTRING_START = "# @LEADING_DOCSTRING_START"
+LEADING_DOCSTRING_END = "# @LEADING_DOCSTRING_END"
+
+
 def extract_function_info(function_code: str) -> Optional[Tuple[str, bool]]:
     """
     Extract function name and async status from Python function code.
@@ -45,18 +53,61 @@ def extract_function_info(function_code: str) -> Optional[Tuple[str, bool]]:
     return None
 
 
-def clean_verifier_code(code: str) -> str:
+def extract_leading_content(code: str) -> Tuple[Optional[str], str]:
+    """
+    Extract leading content (docstrings, imports, etc.) before the main function definition.
+
+    Args:
+        code: The verifier code
+
+    Returns:
+        Tuple of (leading_content or None, function_code)
+    """
+    code = code.strip()
+
+    # Find the first top-level function definition (def or async def at column 0)
+    # We need to find "def " or "async def " that's not indented
+    lines = code.split("\n")
+    func_start_idx = None
+
+    for i, line in enumerate(lines):
+        # Check for unindented def or async def
+        if line.startswith("def ") or line.startswith("async def "):
+            func_start_idx = i
+            break
+
+    if func_start_idx is None or func_start_idx == 0:
+        # No leading content or function not found
+        return (None, code)
+
+    # Everything before the function is leading content
+    leading_lines = lines[:func_start_idx]
+    func_lines = lines[func_start_idx:]
+
+    # Clean up leading content - remove empty lines at the end
+    while leading_lines and not leading_lines[-1].strip():
+        leading_lines.pop()
+
+    if not leading_lines:
+        return (None, code)
+
+    leading_content = "\n".join(leading_lines)
+    function_code = "\n".join(func_lines)
+
+    return (leading_content, function_code)
+
+
+def clean_verifier_code(code: str) -> Tuple[str, Optional[str]]:
     """
-    Clean verifier code by removing markdown code fences and normalizing whitespace.
+    Clean verifier code by removing markdown code fences and extracting leading content.
 
     Args:
         code: Raw verifier code string
 
     Returns:
-        Cleaned code string
+        Tuple of (function code, leading_content or None)
     """
-    # Normalize escaped newlines
-    code = code.replace("\\n", "\n").strip()
+    code = code.strip()
 
     # Remove markdown code fences if present
     if "```" in code:
@@ -64,7 +115,75 @@ def clean_verifier_code(code: str) -> str:
         if fence_blocks:
             code = fence_blocks[0].strip()
 
-    return code
+    # Extract leading content (docstrings, imports, etc.) if present
+    leading_content, code = extract_leading_content(code)
+
+    return (code, leading_content)
+
+
+def format_leading_content_as_comment(content: str) -> str:
+    """
+    Format leading content (docstrings, imports, etc.) as a comment block with markers.
+
+    Args:
+        content: The leading content (docstrings, imports, etc.)
+
+    Returns:
+        Formatted comment block
+    """
+    lines = [LEADING_CONTENT_START]
+
+    for line in content.split("\n"):
+        # Prefix each line with "# |" to preserve exact content including empty lines
+        lines.append(f"# |{line}")
+
+    lines.append(LEADING_CONTENT_END)
+    return "\n".join(lines)
+
+
+def parse_leading_content_from_comments(comment_block: str) -> str:
+    """
+    Parse leading content from a comment block with markers.
+
+    Args:
+        comment_block: The comment block between markers
+
+    Returns:
+        Reconstructed leading content
+    """
+    lines = []
+    for line in comment_block.split("\n"):
+        # Remove "# |" prefix (new format)
+        if line.startswith("# |"):
+            lines.append(line[3:])
+        # Legacy format: "# " prefix
+        elif line.startswith("# "):
+            lines.append(line[2:])
+        elif line == "#":
+            lines.append("")
+
+    return "\n".join(lines)
+
+
+def parse_legacy_docstring_from_comments(comment_block: str) -> str:
+    """
+    Parse a docstring from legacy comment block with markers.
+
+    Args:
+        comment_block: The comment block between markers
+
+    Returns:
+        Reconstructed docstring with triple quotes
+    """
+    lines = []
+    for line in comment_block.split("\n"):
+        # Remove "# " prefix
+        if line.startswith("# "):
+            lines.append(line[2:])
+        elif line == "#":
+            lines.append("")
+
+    return '"""' + "\n".join(lines) + '"""'
 
 
 def extract_verifiers_to_file(json_path: str, py_path: str) -> None:
@@ -123,8 +242,8 @@ def extract_verifiers_to_file(json_path: str, py_path: str) -> None:
             missing_verifier.append(task_key)
             continue
 
-        # Clean the code
-        cleaned_code = clean_verifier_code(verifier_code)
+        # Clean the code and extract leading content (docstrings, imports, etc.)
+        cleaned_code, leading_content = clean_verifier_code(verifier_code)
 
         # Extract function info
         func_info = extract_function_info(cleaned_code)
@@ -142,6 +261,7 @@ def extract_verifiers_to_file(json_path: str, py_path: str) -> None:
                 "function_name": function_name,
                 "is_async": is_async,
                 "code": cleaned_code,
+                "leading_content": leading_content,
             }
         )
 
@@ -195,7 +315,7 @@ def extract_verifiers_to_file(json_path: str, py_path: str) -> None:
         f.write("import json\n")
         f.write("import re\n")
         f.write("import string\n")
-        f.write("from typing import Any\n")
+        f.write("from typing import Any, Dict, List\n")
         f.write("\n")
         f.write("# Helper functions available in verifier namespace\n")
         f.write(
@@ -248,6 +368,11 @@ def extract_verifiers_to_file(json_path: str, py_path: str) -> None:
                 f"# Function: {ver['function_name']} ({'async' if ver['is_async'] else 'sync'})\n"
             )
 
+            # Write leading content (docstrings, imports) as comments if present
+            if ver["leading_content"]:
+                f.write(format_leading_content_as_comment(ver["leading_content"]))
+                f.write("\n")
+
             # Write decorator - use verifier for async, verifier_sync for sync
             decorator_name = "verifier" if ver["is_async"] else "verifier_sync"
             f.write(f'@{decorator_name}(key="{ver["task_key"]}")\n')
@@ -262,7 +387,7 @@ def extract_verifiers_to_file(json_path: str, py_path: str) -> None:
     print(f"  2. Run: python {sys.argv[0]} apply {json_path} {py_path}")
 
 
-def parse_verifiers_from_file(python_path: str) -> Dict[str, str]:
+def parse_verifiers_from_file(python_path: str) -> Dict[str, dict]:
     """
     Parse verifiers from a Python file and extract them by task key.
 
@@ -270,7 +395,7 @@ def parse_verifiers_from_file(python_path: str) -> Dict[str, str]:
         python_path: Path to Python file containing verifiers
 
     Returns:
-        Dictionary mapping task_key to verifier code
+        Dictionary mapping task_key to dict with 'code' and 'leading_content'
     """
     print(f"Reading verifiers from: {python_path}")
 
@@ -281,14 +406,15 @@ def parse_verifiers_from_file(python_path: str) -> Dict[str, str]:
         print(f"✗ Error: File '{python_path}' not found")
         sys.exit(1)
 
-    # Split content by the separator comments to get individual verifier sections
-    # The separator is "# ------------------------------------------------------------------------------"
-    # Each section starts with "# Task: <key>"
-
     verifiers = {}
 
-    # Split by "# Task: " markers to find each verifier block
-    task_blocks = re.split(r"\n# Task: ", content)
+    # Split by "# Task: " markers followed by a task key pattern (uuid or specific format)
+    # This avoids splitting on "# Task: " that appears inside docstring comments
+    # Task keys look like: task_uuid, task_xxx_timestamp_xxx, or send_xxx_xxx
+    task_key_pattern = (
+        r"(?:task_[a-f0-9-]+|task_[a-z0-9]+_\d+_[a-z0-9]+|[a-z_]+_[a-z0-9]+)"
+    )
+    task_blocks = re.split(rf"\n# Task: (?={task_key_pattern})", content)
 
     for block in task_blocks[1:]:  # Skip the first block (header)
         # Extract task key from the first line
@@ -299,6 +425,10 @@ def parse_verifiers_from_file(python_path: str) -> Dict[str, str]:
         # First line should be the task key
         task_key = lines[0].strip()
 
+        # Skip if this doesn't look like a task key (sanity check)
+        if not re.match(task_key_pattern, task_key):
+            continue
+
         # Find the @verifier or @verifier_sync decorator to extract the key parameter
         verifier_match = re.search(
             r'@verifier(?:_sync)?\(key=["\']([^"\']+)["\']\s*(?:,\s*[^)]+)?\)', block
@@ -306,6 +436,26 @@ def parse_verifiers_from_file(python_path: str) -> Dict[str, str]:
         if verifier_match:
             task_key = verifier_match.group(1)
 
+        # Check for leading content markers (new format)
+        leading_content = None
+        if LEADING_CONTENT_START in block:
+            start_idx = block.find(LEADING_CONTENT_START)
+            end_idx = block.find(LEADING_CONTENT_END)
+            if start_idx != -1 and end_idx != -1:
+                comment_block = block[
+                    start_idx + len(LEADING_CONTENT_START) : end_idx
+                ].strip()
+                leading_content = parse_leading_content_from_comments(comment_block)
+        # Fallback: check for legacy docstring markers
+        elif LEADING_DOCSTRING_START in block:
+            start_idx = block.find(LEADING_DOCSTRING_START)
+            end_idx = block.find(LEADING_DOCSTRING_END)
+            if start_idx != -1 and end_idx != -1:
+                comment_block = block[
+                    start_idx + len(LEADING_DOCSTRING_START) : end_idx
+                ].strip()
+                leading_content = parse_legacy_docstring_from_comments(comment_block)
+
         # Find the function definition (async def or def)
         # Extract from the function start until we hit the separator or end
         func_pattern = r"((async\s+)?def\s+\w+.*?)(?=\n# -+\n|\n# Task:|\Z)"
@@ -313,26 +463,31 @@ def parse_verifiers_from_file(python_path: str) -> Dict[str, str]:
 
         if func_match:
             function_code = func_match.group(1).strip()
-            verifiers[task_key] = function_code
+            verifiers[task_key] = {
+                "code": function_code,
+                "leading_content": leading_content,
+            }
 
     # If the above approach didn't work, try a direct pattern match
     if not verifiers:
         # Pattern to match @verifier or @verifier_sync decorator with key and the following function
-        # Look for the decorator, then capture everything until we hit a dedented line or separator
         pattern = r'@verifier(?:_sync)?\(key=["\']([^"\']+)["\']\s*(?:,\s*[^)]+)?\)\s*\n((?:async\s+)?def\s+[^\n]+:(?:\n(?:    |\t).*)*(?:\n(?:    |\t).*)*)'
 
         matches = re.findall(pattern, content, re.MULTILINE)
 
         for task_key, function_code in matches:
-            verifiers[task_key] = function_code.strip()
+            verifiers[task_key] = {
+                "code": function_code.strip(),
+                "leading_content": None,
+            }
 
     print(f"✓ Found {len(verifiers)} verifier(s)")
 
     # Analyze async vs sync
     async_count = 0
     sync_count = 0
-    for code in verifiers.values():
-        func_info = extract_function_info(code)
+    for data in verifiers.values():
+        func_info = extract_function_info(data["code"])
         if func_info:
             _, is_async = func_info
             if is_async:
@@ -346,6 +501,22 @@ def parse_verifiers_from_file(python_path: str) -> Dict[str, str]:
     return verifiers
 
 
+def normalize_code_for_comparison(code: str) -> str:
+    """
+    Normalize code for comparison to avoid false positives.
+    Removes leading/trailing whitespace and normalizes line endings.
+    """
+    # Strip and normalize line endings
+    code = code.strip().replace("\r\n", "\n")
+    # Normalize trailing whitespace on each line
+    lines = code.split("\n")
+    lines = [line.rstrip() for line in lines]
+    code = "\n".join(lines)
+    # Normalize multiple blank lines to single (2+ newlines → 1)
+    code = re.sub(r"\n\n+", "\n", code)
+    return code
+
+
 def apply_verifiers_to_json(json_path: str, python_path: str) -> None:
     """
     Apply verifiers from Python file back into JSON task file (updates in-place).
@@ -386,17 +557,43 @@ def apply_verifiers_to_json(json_path: str, python_path: str) -> None:
             continue
 
         if task_key in verifiers:
-            new_code = verifiers[task_key]
-            
-            # Escape newlines in debug print patterns (>>> and <<<)
-            # These should be \n escape sequences, not actual newlines
-            new_code = new_code.replace(">>>\n", ">>>\\n")
-            new_code = new_code.replace("\n<<<", "\\n<<<")
-            
-            old_code = task.get("verifier_func", "").strip()
+            ver_data = verifiers[task_key]
+
+            # Reconstruct the full verifier code with leading content if present
+            if ver_data["leading_content"]:
+                new_code = ver_data["leading_content"] + "\n" + ver_data["code"]
+            else:
+                new_code = ver_data["code"]
+
+            old_code = task.get("verifier_func", "")
+
+            # Normalize both for comparison
+            old_normalized = normalize_code_for_comparison(old_code)
+            new_normalized = normalize_code_for_comparison(new_code)
+
+            # Debug: show comparison info
+            old_len = len(old_normalized)
+            new_len = len(new_normalized)
+
+            if old_normalized == new_normalized:
+                if old_code != new_code:
+                    print(
+                        f"  [DEBUG] {task_key}: Codes differ in whitespace only (normalized match)"
+                    )
+            else:
+                # Find first difference position for debugging
+                min_len = min(old_len, new_len)
+                diff_pos = min_len
+                for i in range(min_len):
+                    if old_normalized[i] != new_normalized[i]:
+                        diff_pos = i
+                        break
+                print(
+                    f"  [DEBUG] {task_key}: Code changed (old={old_len}, new={new_len}, first_diff@{diff_pos})"
+                )
 
             # Only update if the code actually changed
-            if old_code != new_code.strip():
+            if old_normalized != new_normalized:
                 # Update verifier_func with new code
                 task["verifier_func"] = new_code
 
@@ -451,14 +648,17 @@ def validate_verifiers_file(python_path: str) -> None:
     print("\nValidating verifiers...")
     errors = []
 
-    for task_key, code in verifiers.items():
-        func_info = extract_function_info(code)
+    for task_key, ver_data in verifiers.items():
+        func_info = extract_function_info(ver_data["code"])
         if not func_info:
             errors.append(f"  - {task_key}: Could not extract function info")
         else:
             function_name, is_async = func_info
+            has_leading = (
+                " (has leading content)" if ver_data["leading_content"] else ""
+            )
             print(
-                f"  ✓ {task_key}: {function_name} ({'async' if is_async else 'sync'})"
+                f"  ✓ {task_key}: {function_name} ({'async' if is_async else 'sync'}){has_leading}"
             )
 
     if errors:

fleet/__init__.py

@@ -73,7 +73,7 @@ from . import env
 from . import global_client as _global_client
 from ._async import global_client as _async_global_client
 
-__version__ = "0.2.79"
+__version__ = "0.2.80"
 
 __all__ = [
     # Core classes

fleet/cli.py

@@ -0,0 +1,354 @@
+"""Fleet CLI - Command line interface for Fleet SDK."""
+
+import json
+import os
+import sys
+from typing import List, Optional
+
+try:
+    import typer
+    from rich.console import Console
+    from rich.table import Table
+except ImportError:
+    print(
+        "Error: CLI dependencies not installed.\n"
+        "Install with: pip install 'fleet-python[cli]'",
+        file=sys.stderr,
+    )
+    sys.exit(1)
+
+from .client import Fleet
+from .models import JobCreateRequest
+
+app = typer.Typer(
+    name="flt",
+    help="Fleet CLI - Interact with Fleet jobs and sessions",
+    no_args_is_help=True,
+)
+jobs_app = typer.Typer(help="Manage jobs", no_args_is_help=True)
+sessions_app = typer.Typer(help="Manage sessions", no_args_is_help=True)
+
+app.add_typer(jobs_app, name="jobs")
+app.add_typer(sessions_app, name="sessions")
+
+console = Console()
+
+
+CLI_DEFAULT_BASE_URL = "https://us-west-1.fleetai.com"
+
+
+def get_client() -> Fleet:
+    """Get a Fleet client using environment variables."""
+    api_key = os.getenv("FLEET_API_KEY")
+    if not api_key:
+        console.print(
+            "[red]Error:[/red] FLEET_API_KEY environment variable not set",
+            style="bold",
+        )
+        raise typer.Exit(1)
+    base_url = os.getenv("FLEET_BASE_URL", CLI_DEFAULT_BASE_URL)
+    return Fleet(api_key=api_key, base_url=base_url)
+
+
+# Jobs commands
+
+
+@jobs_app.command("list")
+def list_jobs(
+    team_id: Optional[str] = typer.Option(None, "--team-id", help="Filter by team ID (admin only)"),
+    output_json: bool = typer.Option(False, "--json", help="Output as JSON"),
+):
+    """List all jobs."""
+    client = get_client()
+    jobs = client.list_jobs(team_id=team_id)
+
+    if output_json:
+        console.print(json.dumps([j.model_dump() for j in jobs], indent=2, default=str))
+        return
+
+    if not jobs:
+        console.print("No jobs found.")
+        return
+
+    table = Table(title="Jobs")
+    table.add_column("ID", style="cyan")
+    table.add_column("Name", style="green")
+    table.add_column("Status", style="yellow")
+    table.add_column("Created At", style="dim")
+
+    for job in jobs:
+        table.add_row(
+            job.id,
+            job.name or "-",
+            job.status or "-",
+            job.created_at or "-",
+        )
+
+    console.print(table)
+
+
+@jobs_app.command("create")
+def create_job(
+    model: List[str] = typer.Option(..., "--model", "-m", help="Model in 'provider/model' format (repeatable)"),
+    env_key: Optional[str] = typer.Option(None, "--env-key", "-e", help="Environment key"),
+    project_key: Optional[str] = typer.Option(None, "--project-key", "-p", help="Project key"),
+    task_keys: Optional[List[str]] = typer.Option(None, "--task-key", "-t", help="Task key (repeatable)"),
+    name: Optional[str] = typer.Option(None, "--name", "-n", help="Job name. Supports placeholders: {id} (UUID), {sid} (short UUID), {i} (auto-increment, must be suffix)"),
+    pass_k: int = typer.Option(1, "--pass-k", help="Number of passes"),
+    max_steps: Optional[int] = typer.Option(None, "--max-steps", help="Maximum agent steps"),
+    max_duration: int = typer.Option(60, "--max-duration", help="Timeout in minutes"),
+    max_concurrent: int = typer.Option(30, "--max-concurrent", help="Max concurrent per model"),
+    mode: Optional[str] = typer.Option(None, "--mode", help="Mode: 'tool-use' or 'computer-use'"),
+    system_prompt: Optional[str] = typer.Option(None, "--system-prompt", help="Custom system prompt"),
+    model_prompt: Optional[List[str]] = typer.Option(None, "--model-prompt", help="Per-model prompt in 'provider/model=prompt' format (repeatable)"),
+    byok: Optional[List[str]] = typer.Option(None, "--byok", help="Bring Your Own Key in 'provider=key' format (repeatable)"),
+    byok_ttl: Optional[int] = typer.Option(None, "--byok-ttl", help="TTL for BYOK keys in minutes"),
+    harness: Optional[str] = typer.Option(None, "--harness", help="Harness identifier"),
+    output_json: bool = typer.Option(False, "--json", help="Output as JSON"),
+):
+    """Create a new job.
+
+    Requires --model (repeatable) and exactly one of --env-key, --project-key, or --task-key.
+    """
+    # Validate mutual exclusivity
+    sources = [env_key, project_key, task_keys]
+    specified = sum(1 for s in sources if s)
+    if specified != 1:
+        console.print(
+            "[red]Error:[/red] Exactly one of --env-key, --project-key, or --task-key must be specified",
+            style="bold",
+        )
+        raise typer.Exit(1)
+
+    # Parse model prompts
+    model_prompts = None
+    if model_prompt:
+        model_prompts = {}
+        for mp in model_prompt:
+            if "=" not in mp:
+                console.print(
+                    f"[red]Error:[/red] Invalid --model-prompt format: {mp}. Expected 'provider/model=prompt'",
+                    style="bold",
+                )
+                raise typer.Exit(1)
+            key, value = mp.split("=", 1)
+            model_prompts[key] = value
+
+    # Parse BYOK keys
+    byok_keys = None
+    if byok:
+        byok_keys = {}
+        for b in byok:
+            if "=" not in b:
+                console.print(
+                    f"[red]Error:[/red] Invalid --byok format: {b}. Expected 'provider=key'",
+                    style="bold",
+                )
+                raise typer.Exit(1)
+            provider, key = b.split("=", 1)
+            byok_keys[provider] = key
+
+    client = get_client()
+    result = client.create_job(
+        models=model,
+        name=name,
+        pass_k=pass_k,
+        env_key=env_key,
+        project_key=project_key,
+        task_keys=task_keys,
+        max_steps=max_steps,
+        max_duration_minutes=max_duration,
+        max_concurrent_per_model=max_concurrent,
+        mode=mode,
+        system_prompt=system_prompt,
+        model_prompts=model_prompts,
+        byok_keys=byok_keys,
+        byok_ttl_minutes=byok_ttl,
+        harness=harness,
+    )
+
+    if output_json:
+        console.print(json.dumps(result.model_dump(), indent=2, default=str))
+        return
+
+    console.print(f"[green]Job created successfully![/green]")
+    console.print(f"  Job ID: [cyan]{result.job_id}[/cyan]")
+    console.print(f"  Workflow ID: {result.workflow_job_id}")
+    console.print(f"  Status: [yellow]{result.status}[/yellow]")
+    if result.name:
+        console.print(f"  Name: {result.name}")
+
+
+@jobs_app.command("get")
+def get_job(
+    job_id: str = typer.Argument(..., help="Job ID"),
+    team_id: Optional[str] = typer.Option(None, "--team-id", help="Team ID (admin only)"),
+    output_json: bool = typer.Option(False, "--json", help="Output as JSON"),
+):
+    """Get details for a specific job."""
+    client = get_client()
+    job = client.get_job(job_id, team_id=team_id)
+
+    if output_json:
+        console.print(json.dumps(job.model_dump(), indent=2, default=str))
+        return
+
+    console.print(f"[bold]Job Details[/bold]")
+    console.print(f"  ID: [cyan]{job.id}[/cyan]")
+    console.print(f"  Name: {job.name or '-'}")
+    console.print(f"  Status: [yellow]{job.status or '-'}[/yellow]")
+    console.print(f"  Created At: {job.created_at or '-'}")
+
+
+@jobs_app.command("sessions")
+def list_job_sessions(
+    job_id: str = typer.Argument(..., help="Job ID"),
+    output_json: bool = typer.Option(False, "--json", help="Output as JSON"),
+):
+    """List all sessions for a job, grouped by task."""
+    client = get_client()
+    result = client.list_job_sessions(job_id)
+
+    if output_json:
+        console.print(json.dumps(result.model_dump(), indent=2, default=str))
+        return
+
+    console.print(f"[bold]Sessions for Job:[/bold] [cyan]{result.job_id}[/cyan]")
+    console.print(f"Total Sessions: {result.total_sessions}\n")
+
+    for task_group in result.tasks:
+        task_name = task_group.task.key if task_group.task else task_group.task_id or "Unknown"
+        pass_rate_pct = task_group.pass_rate * 100
+
+        console.print(f"[bold green]Task:[/bold green] {task_name}")
+        console.print(f"  Pass Rate: {task_group.passed_sessions}/{task_group.total_sessions} ({pass_rate_pct:.1f}%)")
+        if task_group.average_score is not None:
+            console.print(f"  Average Score: {task_group.average_score:.2f}")
+
+        table = Table(show_header=True)
+        table.add_column("Session ID", style="cyan")
+        table.add_column("Model", style="blue")
+        table.add_column("Status", style="yellow")
+        table.add_column("Steps")
+        table.add_column("Result")
+
+        for session in task_group.sessions:
+            result_str = "-"
+            if session.verifier_execution:
+                if session.verifier_execution.success:
+                    result_str = "[green]PASS[/green]"
+                    if session.verifier_execution.score is not None:
+                        result_str += f" ({session.verifier_execution.score:.2f})"
+                else:
+                    result_str = "[red]FAIL[/red]"
+
+            table.add_row(
+                session.session_id[:8] + "...",
+                session.model,
+                session.status,
+                str(session.step_count),
+                result_str,
+            )
+
+        console.print(table)
+        console.print()
+
+
+# Sessions commands
+
+
+@sessions_app.command("transcript")
+def get_session_transcript(
+    session_id: str = typer.Argument(..., help="Session ID"),
+    output_json: bool = typer.Option(False, "--json", help="Output as JSON"),
+):
+    """Get the transcript for a session."""
+    client = get_client()
+    result = client.get_session_transcript(session_id)
+
+    if output_json:
+        console.print(json.dumps(result.model_dump(), indent=2, default=str))
+        return
+
+    # Header
+    console.print(f"[bold]Session Transcript[/bold]")
+    console.print()
+
+    # Task info
+    if result.task:
+        console.print(f"[bold]Task:[/bold] {result.task.key}")
+        console.print(f"  Environment: {result.task.env_id}")
+        if result.task.version:
+            console.print(f"  Version: {result.task.version}")
+        console.print()
+        console.print(f"[bold]Prompt:[/bold]")
+        console.print(f"  {result.task.prompt[:200]}{'...' if len(result.task.prompt) > 200 else ''}")
+        console.print()
+
+    # Verifier result
+    if result.verifier_execution:
+        status = "[green]PASS[/green]" if result.verifier_execution.success else "[red]FAIL[/red]"
+        console.print(f"[bold]Verifier Result:[/bold] {status}")
+        if result.verifier_execution.score is not None:
+            console.print(f"  Score: {result.verifier_execution.score}")
+        console.print(f"  Execution Time: {result.verifier_execution.execution_time_ms}ms")
+        console.print()
+
+    # Transcript
+    console.print(f"[bold]Transcript:[/bold] ({len(result.transcript)} messages)")
+    console.print("-" * 60)
+
+    for msg in result.transcript:
+        role_colors = {
+            "user": "green",
+            "assistant": "blue",
+            "tool": "yellow",
+            "system": "magenta",
+        }
+        color = role_colors.get(msg.role, "white")
+        console.print(f"[bold {color}]{msg.role.upper()}:[/bold {color}]")
+
+        # Handle content
+        if isinstance(msg.content, str):
+            # Truncate long content
+            content = msg.content
+            if len(content) > 500:
+                content = content[:500] + "..."
+            console.print(f"  {content}")
+        elif isinstance(msg.content, list):
+            # Multimodal content
+            for item in msg.content:
+                if isinstance(item, dict):
+                    if item.get("type") == "text":
+                        text = item.get("text", "")
+                        if len(text) > 500:
+                            text = text[:500] + "..."
+                        console.print(f"  {text}")
+                    elif item.get("type") == "image_url":
+                        console.print(f"  [dim][Image][/dim]")
+                    elif item.get("type") == "tool_use":
+                        console.print(f"  [dim]Tool: {item.get('name', 'unknown')}[/dim]")
+                    elif item.get("type") == "tool_result":
+                        console.print(f"  [dim]Tool Result[/dim]")
+                else:
+                    console.print(f"  {item}")
+        else:
+            console.print(f"  {msg.content}")
+
+        # Tool calls
+        if msg.tool_calls:
+            for tc in msg.tool_calls:
+                if isinstance(tc, dict):
+                    name = tc.get("function", {}).get("name", tc.get("name", "unknown"))
+                    console.print(f"  [dim]-> Tool call: {name}[/dim]")
+
+        console.print()
+
+
+def main():
+    """Entry point for the CLI."""
+    app()
+
+
+if __name__ == "__main__":
+    main()

fleet/client.py

@@ -38,6 +38,12 @@ from .models import (
     TaskUpdateRequest,
     Run,
     HeartbeatResponse,
+    JobCreateRequest,
+    JobResponse,
+    JobListResponse,
+    JobCreateResponse,
+    JobSessionsResponse,
+    SessionTranscriptResponse,
 )
 from .tasks import Task
 
@@ -1033,6 +1039,136 @@ class Fleet:
         )
         return TaskResponse(**response.json())
 
+    # Jobs API methods
+
+    def list_jobs(self, team_id: Optional[str] = None) -> List[JobResponse]:
+        """List all jobs for the authenticated team.
+
+        Args:
+            team_id: Optional team_id to filter by (admin only)
+
+        Returns:
+            List[JobResponse] containing job information
+        """
+        params = {}
+        if team_id is not None:
+            params["team_id"] = team_id
+
+        response = self.client.request("GET", "/v1/jobs", params=params)
+        job_list = JobListResponse(**response.json())
+        return job_list.jobs
+
+    def create_job(
+        self,
+        models: List[str],
+        name: Optional[str] = None,
+        pass_k: int = 1,
+        env_key: Optional[str] = None,
+        project_key: Optional[str] = None,
+        task_keys: Optional[List[str]] = None,
+        excluded_task_keys: Optional[List[str]] = None,
+        max_steps: Optional[int] = None,
+        max_duration_minutes: int = 60,
+        max_concurrent_per_model: int = 30,
+        mode: Optional[str] = None,
+        system_prompt: Optional[str] = None,
+        model_prompts: Optional[Dict[str, str]] = None,
+        byok_keys: Optional[Dict[str, str]] = None,
+        byok_ttl_minutes: Optional[int] = None,
+        harness: Optional[str] = None,
+    ) -> JobCreateResponse:
+        """Create a new job.
+
+        Args:
+            models: List of model identifiers in "provider/model" format
+            name: Optional job name. Supports placeholders: {id} (UUID), {sid} (short UUID), {i} (auto-increment, must be suffix)
+            pass_k: Number of passes (default: 1)
+            env_key: Environment key (mutually exclusive with project_key/task_keys)
+            project_key: Project key (mutually exclusive with env_key/task_keys)
+            task_keys: Specific task keys (mutually exclusive with env_key/project_key)
+            excluded_task_keys: Task keys to exclude
+            max_steps: Maximum agent steps
+            max_duration_minutes: Timeout in minutes (default: 60)
+            max_concurrent_per_model: Max concurrent per model (default: 30)
+            mode: "tool-use" or "computer-use"
+            system_prompt: Custom system prompt
+            model_prompts: Per-model prompts (model -> prompt)
+            byok_keys: Bring Your Own Keys (provider -> API key)
+            byok_ttl_minutes: TTL for BYOK keys in minutes
+            harness: Harness identifier
+
+        Returns:
+            JobCreateResponse containing job_id, workflow_job_id, status, and name
+        """
+        request = JobCreateRequest(
+            name=name,
+            models=models,
+            pass_k=pass_k,
+            env_key=env_key,
+            project_key=project_key,
+            task_keys=task_keys,
+            excluded_task_keys=excluded_task_keys,
+            max_steps=max_steps,
+            max_duration_minutes=max_duration_minutes,
+            max_concurrent_per_model=max_concurrent_per_model,
+            mode=mode,
+            system_prompt=system_prompt,
+            model_prompts=model_prompts,
+            byok_keys=byok_keys,
+            byok_ttl_minutes=byok_ttl_minutes,
+            harness=harness,
+        )
+
+        response = self.client.request(
+            "POST", "/v1/jobs", json=request.model_dump(exclude_none=True)
+        )
+        return JobCreateResponse(**response.json())
+
+    def get_job(self, job_id: str, team_id: Optional[str] = None) -> JobResponse:
+        """Get a specific job by ID.
+
+        Args:
+            job_id: The job ID
+            team_id: Optional team_id to filter by (admin only)
+
+        Returns:
+            JobResponse containing job information
+        """
+        params = {}
+        if team_id is not None:
+            params["team_id"] = team_id
+
+        response = self.client.request("GET", f"/v1/jobs/{job_id}", params=params)
+        return JobResponse(**response.json())
+
+    # Sessions API methods
+
+    def list_job_sessions(self, job_id: str) -> JobSessionsResponse:
+        """List all sessions for a job, grouped by task.
+
+        Args:
+            job_id: The job ID
+
+        Returns:
+            JobSessionsResponse containing sessions grouped by task with statistics
+        """
+        response = self.client.request("GET", f"/v1/sessions/job/{job_id}")
+        return JobSessionsResponse(**response.json())
+
+    def get_session_transcript(self, session_id: str) -> SessionTranscriptResponse:
+        """Get the transcript for a specific session.
+
+        Args:
+            session_id: The session ID
+
+        Returns:
+            SessionTranscriptResponse containing task, instance, verifier result, and messages
+        """
+        response = self.client.request(
+            "GET", f"/v1/sessions/{session_id}/transcript"
+        )
+        return SessionTranscriptResponse(**response.json())
+
     def _create_verifier_from_data(
         self, verifier_id: str, verifier_key: str, verifier_code: str, verifier_sha: str, verifier_runtime_version: Optional[str] = None
     ) -> "SyncVerifierFunction":

fleet/models.py

@@ -405,3 +405,138 @@ class AccountResponse(BaseModel):
     instance_count: int = Field(..., title="Instance Count")
     profile_id: Optional[str] = Field(None, title="Profile Id")
     profile_name: Optional[str] = Field(None, title="Profile Name")
+
+
+# Jobs and Sessions models
+
+
+class JobCreateRequest(BaseModel):
+    """Request payload for creating a new job.
+
+    The name field supports placeholders:
+    - {id}: Replaced with a full UUID
+    - {sid}: Replaced with the first 8 characters of a UUID (short ID)
+    - {i}: Replaced with auto-incrementing number (must be a suffix, e.g., "job-{i}")
+    """
+
+    name: Optional[str] = Field(None, title="Name", max_length=255)
+    models: List[str] = Field(..., title="Models", min_length=1)
+    pass_k: int = Field(1, title="Pass K", ge=1)
+    env_key: Optional[str] = Field(None, title="Env Key")
+    project_key: Optional[str] = Field(None, title="Project Key")
+    task_keys: Optional[List[str]] = Field(None, title="Task Keys", min_length=1)
+    excluded_task_keys: Optional[List[str]] = Field(None, title="Excluded Task Keys")
+    max_steps: Optional[int] = Field(None, title="Max Steps", ge=1)
+    max_duration_minutes: int = Field(60, title="Max Duration Minutes", ge=1)
+    max_concurrent_per_model: int = Field(30, title="Max Concurrent Per Model", ge=1)
+    mode: Optional[str] = Field(None, title="Mode")
+    system_prompt: Optional[str] = Field(None, title="System Prompt")
+    model_prompts: Optional[Dict[str, str]] = Field(None, title="Model Prompts")
+    byok_keys: Optional[Dict[str, str]] = Field(None, title="BYOK Keys")
+    byok_ttl_minutes: Optional[int] = Field(None, title="BYOK TTL Minutes", ge=1)
+    harness: Optional[str] = Field(None, title="Harness")
+
+
+class JobResponse(BaseModel):
+    """Response for a single job."""
+
+    id: str = Field(..., title="Id")
+    name: Optional[str] = Field(None, title="Name")
+    created_at: Optional[str] = Field(None, title="Created At")
+    status: Optional[str] = Field(None, title="Status")
+
+
+class JobListResponse(BaseModel):
+    """Response for listing jobs."""
+
+    jobs: List[JobResponse] = Field(..., title="Jobs")
+    total: int = Field(..., title="Total")
+
+
+class JobCreateResponse(BaseModel):
+    """Response from creating a job."""
+
+    job_id: str = Field(..., title="Job Id")
+    workflow_job_id: str = Field(..., title="Workflow Job Id")
+    status: str = Field(..., title="Status")
+    name: Optional[str] = Field(None, title="Name")
+
+
+class VerifierExecutionResult(BaseModel):
+    """Verifier execution result for a session."""
+
+    success: bool = Field(..., title="Success")
+    score: Optional[float] = Field(None, title="Score")
+    stdout: Optional[str] = Field(None, title="Stdout")
+    execution_time_ms: int = Field(..., title="Execution Time Ms")
+    result: Optional[Any] = Field(None, title="Result")
+
+
+class SessionInfo(BaseModel):
+    """Session information within a job."""
+
+    session_id: str = Field(..., title="Session Id")
+    instance: Optional[Instance] = Field(None, title="Instance")
+    model: str = Field(..., title="Model")
+    status: str = Field(..., title="Status")
+    created_at: str = Field(..., title="Created At")
+    started_at: Optional[str] = Field(None, title="Started At")
+    ended_at: Optional[str] = Field(None, title="Ended At")
+    step_count: int = Field(..., title="Step Count")
+    verifier_execution: Optional[VerifierExecutionResult] = Field(
+        None, title="Verifier Execution"
+    )
+
+
+class TaskInfo(BaseModel):
+    """Task information for session transcript."""
+
+    key: str = Field(..., title="Key")
+    prompt: str = Field(..., title="Prompt")
+    env_id: str = Field(..., title="Env Id")
+    env_variables: Optional[Dict[str, Any]] = Field(None, title="Env Variables")
+    created_at: Optional[str] = Field(None, title="Created At")
+    version: Optional[str] = Field(None, title="Version")
+    verifier_func: Optional[str] = Field(None, title="Verifier Func")
+    verifier_id: Optional[str] = Field(None, title="Verifier Id")
+    metadata: Optional[Dict[str, Any]] = Field(None, title="Metadata")
+
+
+class TaskSessionGroup(BaseModel):
+    """Sessions grouped by task."""
+
+    task_id: Optional[str] = Field(None, title="Task Id")
+    task: Optional[TaskInfo] = Field(None, title="Task")
+    total_sessions: int = Field(..., title="Total Sessions")
+    passed_sessions: int = Field(..., title="Passed Sessions")
+    pass_rate: float = Field(..., title="Pass Rate")
+    average_score: Optional[float] = Field(None, title="Average Score")
+    sessions: List[SessionInfo] = Field(..., title="Sessions")
+
+
+class JobSessionsResponse(BaseModel):
+    """Response for listing sessions for a job."""
+
+    job_id: str = Field(..., title="Job Id")
+    total_sessions: int = Field(..., title="Total Sessions")
+    tasks: List[TaskSessionGroup] = Field(..., title="Tasks")
+
+
+class TranscriptMessage(BaseModel):
+    """A message in the session transcript."""
+
+    role: str = Field(..., title="Role")
+    content: Any = Field(..., title="Content")
+    tool_calls: Optional[List[Any]] = Field(None, title="Tool Calls")
+    tool_call_id: Optional[str] = Field(None, title="Tool Call Id")
+
+
+class SessionTranscriptResponse(BaseModel):
+    """Response for a session transcript."""
+
+    task: Optional[TaskInfo] = Field(None, title="Task")
+    instance: Optional[Instance] = Field(None, title="Instance")
+    verifier_execution: Optional[VerifierExecutionResult] = Field(
+        None, title="Verifier Execution"
+    )
+    transcript: List[TranscriptMessage] = Field(..., title="Transcript")

{fleet_python-0.2.79.dist-info → fleet_python-0.2.80.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fleet-python
-Version: 0.2.79
+Version: 0.2.80
 Summary: Python SDK for Fleet environments
 Author-email: Fleet AI <nic@fleet.so>
 License: Apache-2.0
@@ -26,6 +26,9 @@ Requires-Dist: httpx-retries>=0.4.0
 Requires-Dist: typing-extensions>=4.0.0
 Requires-Dist: modulegraph2>=0.2.0
 Requires-Dist: cloudpickle==3.1.1
+Provides-Extra: cli
+Requires-Dist: typer>=0.9.0; extra == "cli"
+Requires-Dist: rich>=10.0.0; extra == "cli"
 Provides-Extra: dev
 Requires-Dist: pytest>=7.0.0; extra == "dev"
 Requires-Dist: pytest-asyncio>=0.21.0; extra == "dev"
@@ -35,6 +38,8 @@ Requires-Dist: mypy>=1.0.0; extra == "dev"
 Requires-Dist: ruff>=0.1.0; extra == "dev"
 Requires-Dist: unasync>=0.6.0; extra == "dev"
 Requires-Dist: python-dotenv>=1.1.1; extra == "dev"
+Requires-Dist: typer>=0.9.0; extra == "dev"
+Requires-Dist: rich>=10.0.0; extra == "dev"
 Provides-Extra: playwright
 Requires-Dist: playwright>=1.40.0; extra == "playwright"
 Dynamic: license-file

{fleet_python-0.2.79.dist-info → fleet_python-0.2.80.dist-info}/RECORD

@@ -15,7 +15,7 @@ examples/export_tasks.py,sha256=Sk2Id4i5tZeVs4ZFQoAn-ABFZv7ewyv4pqtZ-Z-tIEI,3574
 examples/fetch_tasks.py,sha256=3A4F0OSUTwUddTOoDNlI-CSxeOnpl3E3zHBh0Qsw9A8,6865
 examples/gemini_example.py,sha256=qj9WDazQTYNiRHNeUg9Tjkp33lJMwbx8gDfpFe1sDQo,16180
 examples/import_tasks.py,sha256=McF5MbHenPZ7HTjQfNoHHevot0EBILMSsWtroicVT30,11619
-examples/iterate_verifiers.py,sha256=GCPrxz-BgEU5yv7cmiLwAo1piILIaYOQaBNON04ATMc,18457
+examples/iterate_verifiers.py,sha256=XbMWyWKAYKJSufVPehG3C_JeSC7ZVXw2_s1KzIMnGVs,25435
 examples/json_tasks_example.py,sha256=CYPESGGtOo0fmsDdLidujTfsE4QlJHw7rOhyVqPJ_Ls,5329
 examples/nova_act_example.py,sha256=rH23Lp74Okf0rn8ynMdWjK2aviEf5NLPH4k_53Pyxho,831
 examples/openai_example.py,sha256=dEWERrTEP5xBiGkLkQjBQGd2NqoxX6gcW6XteBPsWFQ,8231
@@ -23,13 +23,14 @@ examples/openai_simple_example.py,sha256=HmiufucrAZne7tHq9uoEsDWlEhjNC265bQAyIGB
 examples/query_builder_example.py,sha256=-cOMfWGNifYfYEt_Ds73XpwATZvFDL6F4KTkVxdMjzg,3951
 examples/quickstart.py,sha256=1VT39IRRhemsJgxi0O0gprdpcw7HB4pYO97GAYagIcg,3788
 examples/test_cdp_logging.py,sha256=AkCwQCgOTQEI8w3v0knWK_4eXMph7L9x07wj9yIYM10,2836
-fleet/__init__.py,sha256=TBdp5lZtC6ltn7vuRnBNSFcArYy-2v_ONL8M2ZDzxiM,4218
+fleet/__init__.py,sha256=4A84OI2wQx-uXrTMvQ86ue1Su9G3yfhTZCB78mICXZU,4218
 fleet/base.py,sha256=Z0NwvcXx43tu8j_8hmkN4FJHMl1RRYZz4I9F9xN1kkI,9609
-fleet/client.py,sha256=3L-woNgM-pjKgzoXhq2pgY2RHvsgUm1qBvFBkSbMWBo,45077
+fleet/cli.py,sha256=YIPllz1IrdI8U4GFdxlSOmMWE02LhvWqTqnl_ORb8_8,12695
+fleet/client.py,sha256=ZF7-my1yIhR7irVgvZTB-4WrNxS9mb8fJxXqq_wfR1U,50030
 fleet/config.py,sha256=n_wh9Sahu3gGE7nHJ7kqNFUH1qDiBtF4bgZq9MvIBMU,319
 fleet/exceptions.py,sha256=fUmPwWhnT8SR97lYsRq0kLHQHKtSh2eJS0VQ2caSzEI,5055
 fleet/global_client.py,sha256=frrDAFNM2ywN0JHLtlm9qbE1dQpnQJsavJpb7xSR_bU,1072
-fleet/models.py,sha256=7ZRUVBT0_6tiJkotWLTj52oVaeDfP4OsjjkWJxhf8CA,15409
+fleet/models.py,sha256=RlaBzhKxXIYGADN-B-5zujMVP23lhhBXIB6jsJqZ_OU,20726
 fleet/tasks.py,sha256=8pEzXmgC7RslqsMC_0s6shhr_t2WGIRpTRqo-MAQjdg,20778
 fleet/types.py,sha256=L4Y82xICf1tzyCLqhLYUgEoaIIS5h9T05TyFNHSWs3s,652
 fleet/_async/__init__.py,sha256=Ars__cLbW54cO9ccZ5VFX0FJ6zKbaWXs0ByJiZpkjn8,9101
@@ -71,7 +72,7 @@ fleet/verifiers/decorator.py,sha256=RuTjjDijbicNfMSjA7HcTpKueEki5dzNOdTuHS7UoZs,
 fleet/verifiers/parse.py,sha256=qz9AfJrTbjlg-LU-lE8Ciqi7Yt2a8-cs17FdpjTLhMk,8550
 fleet/verifiers/sql_differ.py,sha256=TqTLWyK3uOyLbitT6HYzYEzuSFC39wcyhgk3rcm__k8,6525
 fleet/verifiers/verifier.py,sha256=iqGevW7dSd0J5RdRQjpu-zioy_FYAXnzMfkuB3-QmO0,14601
-fleet_python-0.2.79.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+fleet_python-0.2.80.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
 scripts/fix_sync_imports.py,sha256=X9fWLTpiPGkSHsjyQUDepOJkxOqw1DPj7nd8wFlFqLQ,8368
 scripts/unasync.py,sha256=vWVQxRWX8SRZO5cmzEhpvnG_REhCWXpidIGIpWmEcvI,696
 tests/__init__.py,sha256=Re1SdyxH8NfyL1kjhi7SQkGP1mYeWB-D6UALqdIMd8I,35
@@ -81,7 +82,8 @@ tests/test_instance_dispatch.py,sha256=CvU4C3LBIqsYZdEsEFfontGjyxAZfVYyXnGwxyIvX
 tests/test_sqlite_resource_dual_mode.py,sha256=Mh8jBd-xsIGDYFsOACKKK_5DXMUYlFFS7W-jaY6AjG4,8734
 tests/test_sqlite_shared_memory_behavior.py,sha256=fKx_1BmLS3b8x-9pMgjMycpnaHWY8P-2ZuXEspx6Sbw,4082
 tests/test_verifier_from_string.py,sha256=Lxi3TpFHFb-hG4-UhLKZJkqo84ax9YJY8G6beO-1erM,13581
-fleet_python-0.2.79.dist-info/METADATA,sha256=P2vpxM2ANXpzP5XUM2yv5vrpezwD3Hyhc6WEKa1cNmE,3524
-fleet_python-0.2.79.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-fleet_python-0.2.79.dist-info/top_level.txt,sha256=qb1zIbtEktyhRFZdqVytwg54l64qtoZL0wjHB4bUg3c,29
-fleet_python-0.2.79.dist-info/RECORD,,
+fleet_python-0.2.80.dist-info/METADATA,sha256=HfUrJVpZUNbk7CXH7zDocV8IqIBzY-WGuuwGPp35FeQ,3720
+fleet_python-0.2.80.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+fleet_python-0.2.80.dist-info/entry_points.txt,sha256=qKIQ326cHR5WyCd16QnrW-1DpcT0YyxVRDb3IlTyzTA,39
+fleet_python-0.2.80.dist-info/top_level.txt,sha256=qb1zIbtEktyhRFZdqVytwg54l64qtoZL0wjHB4bUg3c,29
+fleet_python-0.2.80.dist-info/RECORD,,

fleet_python-0.2.80.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+flt = fleet.cli:main