sifr-benchmark 0.1.15__py3-none-any.whl

sifr_benchmark/ground_truth.py

@@ -0,0 +1,250 @@
+"""
+Ground truth generation for agent tasks using GPT-4o Vision.
+Generates tasks with element IDs as answers.
+"""
+
+import base64
+import json
+from pathlib import Path
+
+AGENT_GROUND_TRUTH_PROMPT = """You are analyzing a webpage screenshot alongside its SiFR representation.
+
+SiFR is a compact format describing UI elements. Each element has an ID like btn001, lnk003, inp001.
+
+Your task: Generate agent tasks where the answer is an element ID from the SiFR.
+
+Look at the screenshot to understand WHAT each element does.
+Look at the SiFR to find the correct element ID.
+
+Generate these task types:
+
+1. ACTION_CLICK (3-5 tasks): "What element ID should I click to [action]?"
+   - Login/signup buttons
+   - Navigation links
+   - Submit buttons
+   - Menu items
+
+2. ACTION_INPUT (1-2 tasks): "What element ID should I use to [input action]?"
+   - Search fields
+   - Text inputs
+
+3. ACTION_LOCATE (2-3 tasks): "What element ID contains [content]?"
+   - Main heading
+   - Specific text or logo
+
+Rules:
+- ONLY use element IDs that exist in the SiFR below
+- Each answer must be a single element ID (e.g., "btn001", "lnk007", "inp001")
+- Tasks should be clear and unambiguous
+- Focus on common agent actions: login, search, navigate, submit
+
+Respond ONLY in this JSON format:
+{
+  "page_title": "detected page title",
+  "tasks": [
+    {
+      "id": "act_01",
+      "type": "action_click",
+      "question": "What element ID should I click to login?",
+      "answer": "lnk007",
+      "element_text": "login"
+    },
+    {
+      "id": "act_02", 
+      "type": "action_input",
+      "question": "What element ID should I use to enter a search query?",
+      "answer": "inp001",
+      "element_text": "Search"
+    }
+  ]
+}
+
+SiFR content:
+```
+{sifr_content}
+```
+"""
+
+
+def encode_image(image_path: Path) -> str:
+    """Encode image to base64."""
+    with open(image_path, "rb") as f:
+        return base64.b64encode(f.read()).decode("utf-8")
+
+
+def load_sifr(sifr_path: Path) -> str:
+    """Load SiFR file content."""
+    with open(sifr_path, "r", encoding="utf-8") as f:
+        return f.read()
+
+
+def extract_json(text: str) -> str:
+    """
+    Extract JSON object from text - handles various GPT response formats.
+    """
+    # Try markdown code blocks first
+    if "```json" in text:
+        try:
+            return text.split("```json")[1].split("```")[0].strip()
+        except IndexError:
+            pass
+    
+    if "```" in text:
+        try:
+            return text.split("```")[1].split("```")[0].strip()
+        except IndexError:
+            pass
+    
+    # Find JSON object by matching braces
+    start = text.find("{")
+    if start == -1:
+        return None
+    
+    # Find matching closing brace
+    depth = 0
+    for i, char in enumerate(text[start:], start):
+        if char == "{":
+            depth += 1
+        elif char == "}":
+            depth -= 1
+            if depth == 0:
+                return text[start:i+1]
+    
+    return None
+
+
+def generate_ground_truth(
+    screenshot_path: Path, 
+    sifr_path: Path,
+    output_path: Path = None
+) -> dict:
+    """
+    Generate agent ground truth from screenshot + SiFR.
+    
+    Args:
+        screenshot_path: Path to screenshot PNG
+        sifr_path: Path to SiFR file
+        output_path: Optional path to save ground truth JSON
+        
+    Returns:
+        Ground truth dict with agent tasks
+    """
+    import os
+    from openai import OpenAI
+    
+    api_key = os.getenv("OPENAI_API_KEY")
+    if not api_key:
+        return {"error": "OPENAI_API_KEY not set"}
+    
+    client = OpenAI(api_key=api_key)
+    
+    # Load inputs
+    base64_image = encode_image(screenshot_path)
+    sifr_content = load_sifr(sifr_path)
+    
+    # Check if SiFR is empty
+    if not sifr_content or len(sifr_content.strip()) < 10:
+        return {"error": f"SiFR file is empty or too small: {sifr_path}"}
+    
+    # Truncate SiFR if too large (GPT-4o has ~128K token limit)
+    MAX_SIFR_CHARS = 100000  # ~12K tokens, safe limit
+    if len(sifr_content) > MAX_SIFR_CHARS:
+        # Try to truncate at a sensible point
+        sifr_content = sifr_content[:MAX_SIFR_CHARS]
+        # Find last complete line
+        last_newline = sifr_content.rfind('\n')
+        if last_newline > MAX_SIFR_CHARS * 0.8:
+            sifr_content = sifr_content[:last_newline]
+        sifr_content += "\n... [truncated - showing first 50KB]"
+    
+    # Build prompt with SiFR (use concatenation, not .format() - SiFR contains {})
+    prompt = AGENT_GROUND_TRUTH_PROMPT.replace("{sifr_content}", sifr_content)
+    
+    try:
+        response = client.chat.completions.create(
+            model="gpt-4o",
+            messages=[
+                {
+                    "role": "user",
+                    "content": [
+                        {"type": "text", "text": prompt},
+                        {
+                            "type": "image_url",
+                            "image_url": {
+                                "url": f"data:image/png;base64,{base64_image}",
+                                "detail": "high"
+                            }
+                        }
+                    ]
+                }
+            ],
+            max_tokens=2000,
+            temperature=0
+        )
+        
+        # Parse response
+        content = response.choices[0].message.content
+        
+        # Extract JSON from response - robust parsing
+        json_str = extract_json(content)
+        if not json_str:
+            return {"error": f"Could not extract JSON from response: {content[:100]}..."}
+        
+        ground_truth = json.loads(json_str)
+        ground_truth["_meta"] = {
+            "screenshot": str(screenshot_path),
+            "sifr": str(sifr_path),
+            "model": "gpt-4o",
+            "tokens": response.usage.total_tokens,
+            "mode": "agent"
+        }
+        
+        # Save if output path provided
+        if output_path:
+            output_path.parent.mkdir(parents=True, exist_ok=True)
+            with open(output_path, "w", encoding="utf-8") as f:
+                json.dump(ground_truth, f, indent=2, ensure_ascii=False)
+        
+        return ground_truth
+        
+    except Exception as e:
+        return {"error": str(e)}
+
+
+def generate_ground_truth_for_page(page_name: str, base_dir: Path = None) -> dict:
+    """
+    Generate agent ground truth for a captured page.
+    
+    Args:
+        page_name: Name of the page (e.g., "news_ycombinator_com")
+        base_dir: Base directory with datasets/formats structure
+        
+    Returns:
+        Ground truth dict with agent tasks
+    """
+    if base_dir is None:
+        base_dir = Path(".")
+    
+    screenshot_path = base_dir / "datasets" / "formats" / "screenshots" / f"{page_name}.png"
+    sifr_path = base_dir / "datasets" / "formats" / "sifr" / f"{page_name}.sifr"
+    output_path = base_dir / "benchmark" / "ground-truth" / f"{page_name}.json"
+    
+    if not screenshot_path.exists():
+        return {"error": f"Screenshot not found: {screenshot_path}"}
+    
+    if not sifr_path.exists():
+        return {"error": f"SiFR not found: {sifr_path}"}
+    
+    return generate_ground_truth(screenshot_path, sifr_path, output_path)
+
+
+# CLI support
+if __name__ == "__main__":
+    import sys
+    if len(sys.argv) > 1:
+        page_name = sys.argv[1]
+        base_dir = Path(sys.argv[2]) if len(sys.argv) > 2 else Path(".")
+        result = generate_ground_truth_for_page(page_name, base_dir)
+        print(json.dumps(result, indent=2, ensure_ascii=False))
+    else:
+        print("Usage: python ground_truth.py <page_name> [base_dir]")

sifr_benchmark/models.py

@@ -0,0 +1,110 @@
+"""
+Model API integrations.
+"""
+
+import os
+from typing import Optional
+
+SUPPORTED_MODELS = {
+    # OpenAI
+    "gpt-4o": {"provider": "openai", "model": "gpt-4o"},
+    "gpt-4o-mini": {"provider": "openai", "model": "gpt-4o-mini"},
+    "gpt-4-turbo": {"provider": "openai", "model": "gpt-4-turbo"},
+    # Anthropic
+    "claude-sonnet": {"provider": "anthropic", "model": "claude-sonnet-4-20250514"},
+    "claude-haiku": {"provider": "anthropic", "model": "claude-haiku-4-5-20251001"},
+    "claude-opus": {"provider": "anthropic", "model": "claude-opus-4-20250514"},
+}
+
+
+def query_openai(model_id: str, prompt: str) -> dict:
+    """Query OpenAI API."""
+    try:
+        from openai import OpenAI
+    except ImportError:
+        return {"error": "openai package not installed", "response": None, "tokens": 0}
+    
+    api_key = os.getenv("OPENAI_API_KEY")
+    if not api_key:
+        return {"error": "OPENAI_API_KEY not set", "response": None, "tokens": 0}
+    
+    try:
+        client = OpenAI(api_key=api_key)
+        response = client.chat.completions.create(
+            model=model_id,
+            messages=[{"role": "user", "content": prompt}],
+            temperature=0,
+            max_tokens=500,
+        )
+        
+        return {
+            "response": response.choices[0].message.content,
+            "tokens": response.usage.total_tokens,
+            "error": None,
+        }
+    except Exception as e:
+        return {"error": str(e), "response": None, "tokens": 0}
+
+
+def query_anthropic(model_id: str, prompt: str) -> dict:
+    """Query Anthropic API."""
+    try:
+        from anthropic import Anthropic
+    except ImportError:
+        return {"error": "anthropic package not installed", "response": None, "tokens": 0}
+    
+    api_key = os.getenv("ANTHROPIC_API_KEY")
+    if not api_key:
+        return {"error": "ANTHROPIC_API_KEY not set", "response": None, "tokens": 0}
+    
+    try:
+        client = Anthropic(api_key=api_key)
+        response = client.messages.create(
+            model=model_id,
+            max_tokens=500,
+            messages=[{"role": "user", "content": prompt}],
+        )
+        
+        total_tokens = response.usage.input_tokens + response.usage.output_tokens
+        
+        return {
+            "response": response.content[0].text,
+            "tokens": total_tokens,
+            "error": None,
+        }
+    except Exception as e:
+        return {"error": str(e), "response": None, "tokens": 0}
+
+
+def query_model(model_key: str, prompt: str) -> dict:
+    """
+    Query a model by key.
+    
+    Args:
+        model_key: Key from SUPPORTED_MODELS (e.g., "gpt-4o-mini")
+        prompt: The prompt to send
+        
+    Returns:
+        dict with keys: response, tokens, error
+    """
+    if model_key not in SUPPORTED_MODELS:
+        return {
+            "error": f"Unknown model: {model_key}",
+            "response": None,
+            "tokens": 0,
+        }
+    
+    config = SUPPORTED_MODELS[model_key]
+    provider = config["provider"]
+    model_id = config["model"]
+    
+    if provider == "openai":
+        return query_openai(model_id, prompt)
+    elif provider == "anthropic":
+        return query_anthropic(model_id, prompt)
+    else:
+        return {
+            "error": f"Unknown provider: {provider}",
+            "response": None,
+            "tokens": 0,
+        }

sifr_benchmark/runner.py

@@ -0,0 +1,315 @@
+"""
+Benchmark runner - executes agent tasks across models and formats.
+"""
+
+import json
+import time
+from pathlib import Path
+from typing import Optional
+from dataclasses import dataclass
+
+from .models import query_model, SUPPORTED_MODELS
+from .scoring import score_response
+from .formats import load_format
+
+
+@dataclass
+class TestResult:
+    model: str
+    format: str
+    page_id: str
+    task_id: str
+    run: int
+    response: str
+    expected: str
+    score: float
+    confidence: int
+    tokens: int
+    latency_ms: int
+    error: Optional[str] = None
+
+
+# Agent-focused prompt template
+AGENT_PROMPT = """You are a web automation agent. You need to identify which UI element to interact with.
+
+The webpage is described below in {format_name} format:
+
+{context}
+
+TASK: {question}
+
+Rules:
+- Return ONLY the element ID (like btn001, lnk007, inp001)
+- If multiple elements match, return the most relevant one
+- If no element matches, respond with "none"
+
+Respond in this exact format:
+ANSWER: [element ID]
+CONFIDENCE: [0-100]"""
+
+
+class BenchmarkRunner:
+    """Main benchmark runner for agent tasks."""
+
+    def __init__(
+        self,
+        models: list[str],
+        formats: list[str],
+        pages: Optional[list[str]] = None,
+        runs: int = 1,
+        base_dir: Optional[Path] = None,
+    ):
+        self.models = models
+        self.formats = formats
+        self.pages = pages
+        self.runs = runs
+        self.base_dir = Path(base_dir) if base_dir else Path(".")
+
+        self._validate_config()
+
+    def _validate_config(self):
+        """Validate configuration."""
+        for model in self.models:
+            if model not in SUPPORTED_MODELS:
+                raise ValueError(f"Unknown model: {model}. Supported: {list(SUPPORTED_MODELS.keys())}")
+
+    def _load_ground_truth(self, page_id: str) -> dict:
+        """Load ground truth for a page."""
+        patterns = [
+            self.base_dir / "ground-truth" / f"{page_id}.json",  # New structure
+            self.base_dir / "benchmark" / "ground-truth" / f"{page_id}.json",  # Legacy
+        ]
+        
+        for path in patterns:
+            if path.exists():
+                with open(path, encoding="utf-8") as f:
+                    return json.load(f)
+        
+        return {}
+
+    def _get_tasks_from_ground_truth(self, ground_truth: dict) -> list[dict]:
+        """Extract tasks from ground truth."""
+        # New agent format: tasks array
+        if "tasks" in ground_truth and isinstance(ground_truth["tasks"], list):
+            return ground_truth["tasks"]
+        
+        # Legacy format: convert to tasks
+        legacy_tasks = []
+        
+        if ground_truth.get("title"):
+            legacy_tasks.append({
+                "id": "ext_01",
+                "type": "action_locate",
+                "question": "What element ID contains the page title?",
+                "answer": ground_truth.get("title", "")
+            })
+        
+        if ground_truth.get("primary_button", {}).get("text"):
+            legacy_tasks.append({
+                "id": "act_01",
+                "type": "action_click",
+                "question": f"What element ID should I click to {ground_truth['primary_button']['text']}?",
+                "answer": ""  # Unknown in legacy format
+            })
+        
+        return legacy_tasks
+
+    def _discover_pages(self) -> list[str]:
+        """Discover available test pages."""
+        if self.pages:
+            return self.pages
+
+        # New structure: base_dir/ground-truth/
+        gt_path = self.base_dir / "ground-truth"
+        if gt_path.exists():
+            pages = [f.stem for f in gt_path.glob("*.json")]
+            if pages:
+                return pages
+        
+        # Fallback: look for SiFR files
+        sifr_path = self.base_dir / "captures" / "sifr"
+        if sifr_path.exists():
+            pages = [f.stem for f in sifr_path.glob("*.sifr")]
+            if pages:
+                return pages
+
+        return []
+
+    def _build_prompt(self, task: dict, context: str, format_name: str) -> str:
+        """Build prompt for agent task."""
+        return AGENT_PROMPT.format(
+            format_name=format_name,
+            context=context,
+            question=task["question"]
+        )
+
+    def _parse_response(self, raw: str) -> dict:
+        """Parse model response."""
+        result = {"answer": "", "confidence": 50}
+
+        for line in raw.split("\n"):
+            line = line.strip()
+            if line.upper().startswith("ANSWER:"):
+                result["answer"] = line.split(":", 1)[1].strip()
+            elif line.upper().startswith("CONFIDENCE:"):
+                try:
+                    result["confidence"] = int(line.split(":", 1)[1].strip())
+                except ValueError:
+                    pass
+
+        # Fallback: try to extract element ID from raw response
+        if not result["answer"]:
+            import re
+            ids = re.findall(r'\b([a-z]{2,4}\d{2,4})\b', raw.lower())
+            if ids:
+                result["answer"] = ids[0]
+            else:
+                result["answer"] = raw.strip()[:100]
+
+        return result
+
+    def run_single(
+        self,
+        model: str,
+        format_name: str,
+        page_id: str,
+        task: dict,
+        run_num: int,
+    ) -> TestResult:
+        """Run a single test."""
+
+        try:
+            context = load_format(page_id, format_name, self.base_dir)
+        except FileNotFoundError as e:
+            return TestResult(
+                model=model,
+                format=format_name,
+                page_id=page_id,
+                task_id=task["id"],
+                run=run_num,
+                response="",
+                expected=task.get("answer", ""),
+                score=0.0,
+                confidence=0,
+                tokens=0,
+                latency_ms=0,
+                error=f"Format file not found: {format_name}/{page_id}",
+            )
+
+        prompt = self._build_prompt(task, context, format_name)
+
+        start = time.time()
+        response_data = query_model(model, prompt)
+        latency = int((time.time() - start) * 1000)
+
+        if response_data.get("error"):
+            return TestResult(
+                model=model,
+                format=format_name,
+                page_id=page_id,
+                task_id=task["id"],
+                run=run_num,
+                response="",
+                expected=task.get("answer", ""),
+                score=0.0,
+                confidence=0,
+                tokens=response_data.get("tokens", 0),
+                latency_ms=latency,
+                error=response_data["error"],
+            )
+
+        parsed = self._parse_response(response_data["response"])
+        expected = task.get("answer", "")
+        element_text = task.get("element_text", "")  # For HTML/AXTree fallback
+        
+        score = score_response(
+            parsed["answer"],
+            expected,
+            task.get("type", "action"),
+            element_text
+        )
+
+        return TestResult(
+            model=model,
+            format=format_name,
+            page_id=page_id,
+            task_id=task["id"],
+            run=run_num,
+            response=parsed["answer"],
+            expected=expected,
+            score=score,
+            confidence=parsed["confidence"],
+            tokens=response_data.get("tokens", 0),
+            latency_ms=latency,
+        )
+
+    def run(self) -> list[dict]:
+        """Run full benchmark."""
+        pages = self._discover_pages()
+        results = []
+
+        if not pages:
+            print("No pages found for benchmark")
+            return results
+
+        for page_id in pages:
+            ground_truth = self._load_ground_truth(page_id)
+            tasks = self._get_tasks_from_ground_truth(ground_truth)
+            
+            if not tasks:
+                print(f"  ⚠️ No tasks for {page_id}")
+                continue
+
+            for model in self.models:
+                for format_name in self.formats:
+                    for task in tasks:
+                        for run_num in range(1, self.runs + 1):
+                            result = self.run_single(
+                                model=model,
+                                format_name=format_name,
+                                page_id=page_id,
+                                task=task,
+                                run_num=run_num,
+                            )
+                            results.append(result.__dict__)
+                            
+                            # Rate limiting
+                            time.sleep(0.3)
+
+        return results
+
+    def aggregate(self, results: list[dict]) -> list[dict]:
+        """Aggregate results by format (across models)."""
+        agg = {}
+
+        for r in results:
+            if r.get("error"):
+                continue
+
+            key = r["format"]
+            if key not in agg:
+                agg[key] = {
+                    "format": r["format"],
+                    "scores": [],
+                    "tokens": [],
+                    "latencies": [],
+                }
+
+            agg[key]["scores"].append(r["score"])
+            agg[key]["tokens"].append(r["tokens"])
+            agg[key]["latencies"].append(r["latency_ms"])
+
+        summary = []
+        for data in agg.values():
+            scores = data["scores"]
+            tokens = data["tokens"]
+            latencies = data["latencies"]
+
+            summary.append({
+                "format": data["format"],
+                "accuracy": f"{(sum(scores) / len(scores) * 100):.1f}%" if scores else "N/A",
+                "avg_tokens": int(sum(tokens) / len(tokens)) if tokens else 0,
+                "avg_latency": f"{int(sum(latencies) / len(latencies))}ms" if latencies else "N/A",
+            })
+
+        return sorted(summary, key=lambda x: float(x["accuracy"].rstrip("%") or 0), reverse=True)