tsugite-cli 0.3.3__py3-none-any.whl

tsugite/attachments/storage.py

@@ -0,0 +1,178 @@
+"""Attachment management for reusable context."""
+
+import json
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Dict, Optional, Tuple
+
+from tsugite.xdg import get_xdg_config_path
+
+
+def get_attachments_path() -> Path:
+    """Get path to attachments.json file.
+
+    Returns:
+        Path to attachments.json in tsugite config directory
+    """
+    return get_xdg_config_path("attachments.json")
+
+
+def load_attachments() -> Dict[str, Dict[str, str]]:
+    """Load attachments from JSON file.
+
+    Returns:
+        Dictionary of attachments, empty dict if file doesn't exist
+    """
+    attachments_path = get_attachments_path()
+
+    if not attachments_path.exists():
+        return {}
+
+    try:
+        with open(attachments_path, "r", encoding="utf-8") as f:
+            data = json.load(f)
+            return data.get("attachments", {})
+    except (json.JSONDecodeError, IOError) as e:
+        raise RuntimeError(f"Failed to load attachments from {attachments_path}: {e}")
+
+
+def save_attachments(attachments: Dict[str, Dict[str, str]]) -> None:
+    """Save attachments to JSON file.
+
+    Args:
+        attachments: Dictionary of attachment data to save
+
+    Raises:
+        RuntimeError: If save fails
+    """
+    attachments_path = get_attachments_path()
+
+    # Ensure directory exists
+    attachments_path.parent.mkdir(parents=True, exist_ok=True)
+
+    data = {"attachments": attachments}
+
+    try:
+        with open(attachments_path, "w", encoding="utf-8") as f:
+            json.dump(data, f, indent=2, ensure_ascii=False)
+    except IOError as e:
+        raise RuntimeError(f"Failed to save attachments to {attachments_path}: {e}")
+
+
+def add_attachment(alias: str, source: str, content: Optional[str] = None) -> None:
+    """Add or update an attachment.
+
+    For inline text (stdin), provide both source="inline" and content.
+    For file/URL references, provide only source (content will be fetched on demand).
+
+    Args:
+        alias: Unique identifier for the attachment
+        source: Source reference (file path, URL, or "inline" for text)
+        content: Text content (only for inline attachments)
+
+    Raises:
+        ValueError: If alias is empty or invalid parameters
+        RuntimeError: If save fails
+    """
+    if not alias or not alias.strip():
+        raise ValueError("Attachment alias cannot be empty")
+
+    # Validate inline vs reference
+    is_inline = source.lower() in ("inline", "text")
+    if is_inline and not content:
+        raise ValueError("Inline attachments require content")
+
+    attachments = load_attachments()
+    now = datetime.now(timezone.utc).isoformat()
+
+    # Build attachment entry
+    entry = {
+        "source": source,
+        "updated": now,
+    }
+
+    # Only store content for inline attachments
+    if is_inline:
+        entry["content"] = content
+
+    # Add created timestamp for new attachments
+    if alias not in attachments:
+        entry["created"] = now
+    else:
+        # Preserve original created timestamp
+        entry["created"] = attachments[alias].get("created", now)
+
+    attachments[alias] = entry
+    save_attachments(attachments)
+
+
+def get_attachment(alias: str) -> Optional[Tuple[str, Optional[str]]]:
+    """Get an attachment by alias.
+
+    Args:
+        alias: Attachment identifier
+
+    Returns:
+        Tuple of (source, content) if found, None otherwise.
+        For inline attachments, content is the stored text.
+        For file/URL references, content is None (fetch on demand).
+    """
+    attachments = load_attachments()
+
+    if alias not in attachments:
+        return None
+
+    attachment = attachments[alias]
+    source = attachment["source"]
+    content = attachment.get("content")  # None for references
+    return source, content
+
+
+def list_attachments() -> Dict[str, Dict[str, str]]:
+    """List all attachments.
+
+    Returns:
+        Dictionary of all attachment data
+    """
+    return load_attachments()
+
+
+def remove_attachment(alias: str) -> bool:
+    """Remove an attachment.
+
+    Args:
+        alias: Attachment identifier to remove
+
+    Returns:
+        True if attachment was removed, False if it didn't exist
+
+    Raises:
+        RuntimeError: If save fails
+    """
+    attachments = load_attachments()
+
+    if alias not in attachments:
+        return False
+
+    del attachments[alias]
+    save_attachments(attachments)
+    return True
+
+
+def search_attachments(query: str) -> Dict[str, Dict[str, str]]:
+    """Search attachments by alias or source.
+
+    Args:
+        query: Search term (case-insensitive)
+
+    Returns:
+        Dictionary of matching attachments
+    """
+    attachments = load_attachments()
+    query_lower = query.lower()
+
+    return {
+        alias: data
+        for alias, data in attachments.items()
+        if query_lower in alias.lower() or query_lower in data.get("source", "").lower()
+    }

tsugite/attachments/url.py

@@ -0,0 +1,59 @@
+"""Generic URL handler for HTTP/HTTPS attachments."""
+
+import urllib.request
+
+from tsugite.attachments.base import AttachmentHandler
+
+
+class GenericURLHandler(AttachmentHandler):
+    """Handler for generic HTTP(S) URLs."""
+
+    def can_handle(self, source: str) -> bool:
+        """Check if source is an HTTP(S) URL.
+
+        Args:
+            source: Source string
+
+        Returns:
+            True if source is HTTP or HTTPS URL
+        """
+        return source.startswith("http://") or source.startswith("https://")
+
+    def fetch(self, source: str) -> str:
+        """Fetch URL content and convert HTML to markdown if needed.
+
+        Args:
+            source: URL to fetch
+
+        Returns:
+            Content as plain text (HTML converted to markdown if applicable)
+
+        Raises:
+            ValueError: If fetch fails
+        """
+        try:
+            # Fetch URL - get both content and headers
+            with urllib.request.urlopen(source, timeout=30) as response:
+                content_type = response.headers.get("Content-Type", "").lower()
+                content = response.read().decode("utf-8")
+
+            # If HTML, convert to markdown
+            if "text/html" in content_type:
+                try:
+                    import html2text
+
+                    h = html2text.HTML2Text()
+                    h.ignore_links = False
+                    h.ignore_images = False
+                    h.body_width = 0  # Don't wrap lines
+                    return h.handle(content)
+                except ImportError:
+                    # Fall back to raw HTML if html2text not available
+                    # Note: html2text is optional for better readability
+                    return content
+            else:
+                # Plain text, JSON, XML, etc
+                return content
+
+        except Exception as e:
+            raise ValueError(f"Failed to fetch URL '{source}': {e}")

tsugite/attachments/youtube.py

@@ -0,0 +1,101 @@
+"""YouTube transcript handler for attachments."""
+
+import re
+from typing import Optional
+
+from tsugite.attachments.base import AttachmentHandler
+
+
+class YouTubeHandler(AttachmentHandler):
+    """Handler for YouTube video transcripts."""
+
+    def can_handle(self, source: str) -> bool:
+        """Check if source is a YouTube URL.
+
+        Args:
+            source: Source string
+
+        Returns:
+            True if source is a YouTube URL or youtube: prefix
+        """
+        patterns = [
+            r"youtube\.com/watch",
+            r"youtu\.be/",
+            r"^youtube:",
+        ]
+        return any(re.search(pattern, source) for pattern in patterns)
+
+    def fetch(self, source: str) -> str:
+        """Fetch YouTube transcript.
+
+        Args:
+            source: YouTube URL or youtube:VIDEO_ID
+
+        Returns:
+            Transcript as formatted text
+
+        Raises:
+            ValueError: If transcript cannot be fetched
+        """
+        try:
+            from youtube_transcript_api import YouTubeTranscriptApi
+        except ImportError:
+            raise ValueError("youtube-transcript-api not installed. Install with: uv add youtube-transcript-api")
+
+        # Extract video ID
+        video_id = self._extract_video_id(source)
+        if not video_id:
+            raise ValueError(f"Could not extract video ID from: {source}")
+
+        try:
+            # Fetch transcript
+            transcript = YouTubeTranscriptApi.get_transcript(video_id)  # pylint: disable=no-member
+
+            # Format as text
+            lines = []
+            for entry in transcript:
+                timestamp = self._format_timestamp(entry["start"])
+                text = entry["text"]
+                lines.append(f"[{timestamp}] {text}")
+
+            return "\n".join(lines)
+        except Exception as e:
+            raise ValueError(f"Failed to fetch YouTube transcript for {video_id}: {e}")
+
+    def _extract_video_id(self, source: str) -> Optional[str]:
+        """Extract video ID from YouTube URL or youtube: prefix.
+
+        Args:
+            source: YouTube URL or youtube:VIDEO_ID
+
+        Returns:
+            Video ID or None if not found
+        """
+        # Handle youtube:VIDEO_ID format
+        if source.startswith("youtube:"):
+            return source[8:]  # Remove "youtube:" prefix
+
+        # Handle youtu.be/VIDEO_ID format
+        match = re.search(r"youtu\.be/([a-zA-Z0-9_-]+)", source)
+        if match:
+            return match.group(1)
+
+        # Handle youtube.com/watch?v=VIDEO_ID format
+        match = re.search(r"[?&]v=([a-zA-Z0-9_-]+)", source)
+        if match:
+            return match.group(1)
+
+        return None
+
+    def _format_timestamp(self, seconds: float) -> str:
+        """Format seconds as MM:SS timestamp.
+
+        Args:
+            seconds: Time in seconds
+
+        Returns:
+            Formatted timestamp string
+        """
+        minutes = int(seconds // 60)
+        secs = int(seconds % 60)
+        return f"{minutes:02d}:{secs:02d}"

tsugite/benchmark/__init__.py

@@ -0,0 +1,62 @@
+"""Tsugite benchmark framework for evaluating agent performance across different models."""
+
+from .config import (
+    COST_TIERS,
+    EVALUATION_WEIGHTS,
+    MODEL_COSTS,
+    PERFORMANCE_TIERS,
+    SIMILARITY_THRESHOLDS,
+    TEST_CATEGORIES,
+    get_cost_tier,
+    get_performance_tier,
+)
+from .core import BenchmarkConfig, BenchmarkResult, BenchmarkRunner
+from .discovery import BenchmarkTest, TestCase, TestDiscovery
+from .evaluators import (
+    CorrectnessEvaluator,
+    CostEvaluator,
+    LLMEvaluator,
+    PerformanceEvaluator,
+    QualityEvaluator,
+)
+from .execution import TestExecutor
+from .metrics import (
+    BenchmarkMetrics,
+    BenchmarkTestResult,
+    ModelPerformance,
+)
+from .reports import ReportGenerator
+
+__all__ = [
+    # Core
+    "BenchmarkRunner",
+    "BenchmarkResult",
+    "BenchmarkConfig",
+    # Discovery
+    "TestDiscovery",
+    "BenchmarkTest",
+    "TestCase",
+    # Execution
+    "TestExecutor",
+    # Evaluators
+    "CorrectnessEvaluator",
+    "PerformanceEvaluator",
+    "QualityEvaluator",
+    "CostEvaluator",
+    "LLMEvaluator",
+    # Metrics
+    "BenchmarkMetrics",
+    "BenchmarkTestResult",
+    "ModelPerformance",
+    # Reports
+    "ReportGenerator",
+    # Config
+    "SIMILARITY_THRESHOLDS",
+    "PERFORMANCE_TIERS",
+    "EVALUATION_WEIGHTS",
+    "MODEL_COSTS",
+    "COST_TIERS",
+    "TEST_CATEGORIES",
+    "get_performance_tier",
+    "get_cost_tier",
+]

tsugite/benchmark/config.py

@@ -0,0 +1,183 @@
+"""Benchmark configuration and shared constants."""
+
+from dataclasses import dataclass, field
+from typing import Dict
+
+
+@dataclass
+class SimilarityThresholds:
+    """Thresholds for similarity-based evaluation."""
+
+    string_high_similarity: float = 0.9  # High similarity for string matching
+    code_similarity: float = 0.85  # Code similarity threshold
+    json_similarity: float = 0.8  # JSON structure similarity
+    behavior_pass_threshold: float = 0.6  # Minimum score for behavior tests
+    custom_criteria_threshold: float = 0.7  # Minimum for custom criteria
+    llm_evaluation_threshold: float = 0.7  # Minimum LLM judge score
+
+
+@dataclass
+class PerformanceTiers:
+    """Performance tier thresholds."""
+
+    excellent: float = 0.9
+    good: float = 0.75
+    fair: float = 0.6
+    poor: float = 0.4
+    # Below poor is "Very Poor"
+
+
+@dataclass
+class EvaluationWeights:
+    """Weights for blending different evaluation scores."""
+
+    llm_weight: float = 0.4  # Weight for LLM evaluation in blended score
+    base_weight: float = 0.6  # Weight for base evaluation
+    planning_weight: float = 0.3  # Weight for planning evaluation
+    planning_minimum: float = 0.4  # Minimum planning score required
+
+
+@dataclass
+class BehaviorScores:
+    """Score values for behavior-based evaluation."""
+
+    tool_usage: float = 0.3
+    file_created: float = 0.3
+
+
+@dataclass
+class ModelCosts:
+    """Token costs for different model providers (USD per token)."""
+
+    # OpenAI
+    gpt_4: float = 0.00003
+    gpt_4_turbo: float = 0.00001
+    gpt_3_5_turbo: float = 0.000002
+
+    # Anthropic
+    claude_3: float = 0.000015
+    claude_3_5: float = 0.000015
+
+    # Google
+    gemini_pro: float = 0.0000005
+
+    # Local models
+    ollama: float = 0.0
+
+    # Default fallback
+    default: float = 0.00001
+
+    def get_cost_for_model(self, model: str) -> float:
+        """Get cost per token for a model string."""
+        model_lower = model.lower()
+
+        # OpenAI models
+        if "gpt-4-turbo" in model_lower or "gpt-4o" in model_lower:
+            return self.gpt_4_turbo
+        elif "gpt-4" in model_lower:
+            return self.gpt_4
+        elif "gpt-3.5" in model_lower or "gpt-35" in model_lower:
+            return self.gpt_3_5_turbo
+
+        # Anthropic models
+        elif "claude-3-5" in model_lower or "claude-3.5" in model_lower:
+            return self.claude_3_5
+        elif "claude-3" in model_lower:
+            return self.claude_3
+
+        # Google models
+        elif "gemini" in model_lower:
+            return self.gemini_pro
+
+        # Local models
+        elif "ollama" in model_lower or "local" in model_lower:
+            return self.ollama
+
+        return self.default
+
+
+@dataclass
+class CostTiers:
+    """Thresholds for cost tier classification."""
+
+    very_low: float = 0.001
+    low: float = 0.01
+    medium: float = 0.1
+    high: float = 1.0
+    # Above high is "Very High", free is 0.0
+
+
+@dataclass
+class TestCategories:
+    """Known test categories and their prefixes."""
+
+    categories: Dict[str, str] = field(
+        default_factory=lambda: {
+            "basic": "basic_",
+            "tools": "tools_",
+            "scenarios": "scenarios_",
+            "performance": "performance_",
+            "complex": "complex_",
+        }
+    )
+
+    def get_category(self, test_id: str) -> str:
+        """Extract category from test ID."""
+        for category, prefix in self.categories.items():
+            if test_id.startswith(prefix):
+                return category
+        return "unknown"
+
+
+# Global configuration instances
+SIMILARITY_THRESHOLDS = SimilarityThresholds()
+PERFORMANCE_TIERS = PerformanceTiers()
+EVALUATION_WEIGHTS = EvaluationWeights()
+BEHAVIOR_SCORES = BehaviorScores()
+MODEL_COSTS = ModelCosts()
+COST_TIERS = CostTiers()
+TEST_CATEGORIES = TestCategories()
+
+
+def get_performance_tier(accuracy: float) -> str:
+    """Get performance tier label based on accuracy.
+
+    Args:
+        accuracy: Accuracy score from 0.0 to 1.0
+
+    Returns:
+        Performance tier label
+    """
+    if accuracy >= PERFORMANCE_TIERS.excellent:
+        return "Excellent"
+    elif accuracy >= PERFORMANCE_TIERS.good:
+        return "Good"
+    elif accuracy >= PERFORMANCE_TIERS.fair:
+        return "Fair"
+    elif accuracy >= PERFORMANCE_TIERS.poor:
+        return "Poor"
+    else:
+        return "Very Poor"
+
+
+def get_cost_tier(cost: float) -> str:
+    """Get cost tier label based on cost in USD.
+
+    Args:
+        cost: Cost in USD
+
+    Returns:
+        Cost tier label
+    """
+    if cost == 0:
+        return "Free"
+    elif cost < COST_TIERS.very_low:
+        return "Very Low"
+    elif cost < COST_TIERS.low:
+        return "Low"
+    elif cost < COST_TIERS.medium:
+        return "Medium"
+    elif cost < COST_TIERS.high:
+        return "High"
+    else:
+        return "Very High"