tech-hub-skills 1.2.0 → 1.5.2

package/.claude/roles/ai-engineer/skills/01-prompt-engineering/prompt_ab_tester.py

@@ -0,0 +1,356 @@
+"""
+Prompt A/B Testing Framework
+Compare prompt variations with statistical significance testing.
+"""
+
+import asyncio
+import json
+from datetime import datetime
+from typing import List, Dict, Any, Optional, Callable
+from dataclasses import dataclass, asdict
+from scipy import stats
+import numpy as np
+import pandas as pd
+
+
+@dataclass
+class PromptVariant:
+    """A prompt variant for A/B testing."""
+    id: str
+    name: str
+    template: str
+    metadata: Dict[str, Any] = None
+
+    def __post_init__(self):
+        if self.metadata is None:
+            self.metadata = {}
+
+
+@dataclass
+class TestResult:
+    """Result from testing a single variant."""
+    variant_id: str
+    response: str
+    latency_ms: float
+    tokens_used: int
+    cost: float
+    quality_score: Optional[float] = None
+    timestamp: str = None
+
+    def __post_init__(self):
+        if self.timestamp is None:
+            self.timestamp = datetime.now().isoformat()
+
+
+class ABTest:
+    """A/B test experiment for prompt variants."""
+
+    def __init__(
+        self,
+        name: str,
+        variants: List[PromptVariant],
+        evaluation_fn: Optional[Callable] = None
+    ):
+        self.name = name
+        self.variants = {v.id: v for v in variants}
+        self.evaluation_fn = evaluation_fn or self._default_evaluation
+        self.results: List[TestResult] = []
+
+    def add_result(self, result: TestResult) -> None:
+        """Add a test result."""
+        self.results.append(result)
+
+    def get_variant_results(self, variant_id: str) -> List[TestResult]:
+        """Get all results for a specific variant."""
+        return [r for r in self.results if r.variant_id == variant_id]
+
+    def calculate_metrics(self, variant_id: str) -> Dict[str, float]:
+        """Calculate aggregate metrics for a variant."""
+        results = self.get_variant_results(variant_id)
+
+        if not results:
+            return {}
+
+        return {
+            "sample_size": len(results),
+            "avg_latency_ms": np.mean([r.latency_ms for r in results]),
+            "avg_cost": np.mean([r.cost for r in results]),
+            "avg_tokens": np.mean([r.tokens_used for r in results]),
+            "avg_quality_score": np.mean([r.quality_score for r in results if r.quality_score is not None]),
+            "p50_latency": np.percentile([r.latency_ms for r in results], 50),
+            "p95_latency": np.percentile([r.latency_ms for r in results], 95),
+            "p99_latency": np.percentile([r.latency_ms for r in results], 99),
+        }
+
+    def compare_variants(
+        self,
+        variant_a_id: str,
+        variant_b_id: str,
+        metric: str = "quality_score"
+    ) -> Dict[str, Any]:
+        """
+        Compare two variants with statistical significance testing.
+
+        Args:
+            variant_a_id: First variant ID
+            variant_b_id: Second variant ID
+            metric: Metric to compare ('quality_score', 'latency_ms', 'cost')
+
+        Returns:
+            Comparison results with p-value and effect size
+        """
+        results_a = self.get_variant_results(variant_a_id)
+        results_b = self.get_variant_results(variant_b_id)
+
+        if not results_a or not results_b:
+            return {"error": "Insufficient data for comparison"}
+
+        # Extract metric values
+        values_a = [getattr(r, metric) for r in results_a if getattr(r, metric) is not None]
+        values_b = [getattr(r, metric) for r in results_b if getattr(r, metric) is not None]
+
+        if not values_a or not values_b:
+            return {"error": f"No valid {metric} data"}
+
+        # Perform t-test
+        t_stat, p_value = stats.ttest_ind(values_a, values_b)
+
+        # Calculate effect size (Cohen's d)
+        pooled_std = np.sqrt((np.std(values_a)**2 + np.std(values_b)**2) / 2)
+        cohens_d = (np.mean(values_a) - np.mean(values_b)) / pooled_std if pooled_std > 0 else 0
+
+        # Determine winner
+        if p_value < 0.05:
+            if metric in ["latency_ms", "cost"]:
+                # Lower is better
+                winner = variant_a_id if np.mean(values_a) < np.mean(values_b) else variant_b_id
+            else:
+                # Higher is better
+                winner = variant_a_id if np.mean(values_a) > np.mean(values_b) else variant_b_id
+            significant = True
+        else:
+            winner = "No significant difference"
+            significant = False
+
+        return {
+            "variant_a": {
+                "id": variant_a_id,
+                "mean": float(np.mean(values_a)),
+                "std": float(np.std(values_a)),
+                "sample_size": len(values_a)
+            },
+            "variant_b": {
+                "id": variant_b_id,
+                "mean": float(np.mean(values_b)),
+                "std": float(np.std(values_b)),
+                "sample_size": len(values_b)
+            },
+            "t_statistic": float(t_stat),
+            "p_value": float(p_value),
+            "cohens_d": float(cohens_d),
+            "significant": significant,
+            "winner": winner,
+            "metric": metric,
+            "improvement": float((np.mean(values_b) - np.mean(values_a)) / np.mean(values_a) * 100)
+        }
+
+    def generate_report(self) -> str:
+        """Generate a comprehensive A/B test report."""
+        lines = [
+            f"📊 A/B Test Report: {self.name}",
+            "=" * 80,
+            f"Total Results: {len(self.results)}",
+            f"Variants Tested: {len(self.variants)}",
+            "\n"
+        ]
+
+        # Metrics for each variant
+        lines.append("📈 Variant Performance:")
+        for variant_id, variant in self.variants.items():
+            metrics = self.calculate_metrics(variant_id)
+            if metrics:
+                lines.append(f"\n  {variant.name} (ID: {variant_id})")
+                lines.append(f"    Sample Size: {metrics['sample_size']}")
+                lines.append(f"    Avg Quality Score: {metrics['avg_quality_score']:.2f}")
+                lines.append(f"    Avg Latency: {metrics['avg_latency_ms']:.0f}ms (p95: {metrics['p95_latency']:.0f}ms)")
+                lines.append(f"    Avg Cost: ${metrics['avg_cost']:.4f}")
+                lines.append(f"    Avg Tokens: {metrics['avg_tokens']:.0f}")
+
+        # Statistical comparisons
+        if len(self.variants) == 2:
+            variant_ids = list(self.variants.keys())
+            comparison = self.compare_variants(variant_ids[0], variant_ids[1])
+
+            lines.append("\n\n🔬 Statistical Comparison:")
+            lines.append(f"  Metric: {comparison.get('metric', 'N/A')}")
+            lines.append(f"  P-Value: {comparison.get('p_value', 0):.4f}")
+            lines.append(f"  Cohen's d: {comparison.get('cohens_d', 0):.3f}")
+            lines.append(f"  Significant: {'Yes' if comparison.get('significant') else 'No'}")
+            lines.append(f"  Winner: {comparison.get('winner', 'N/A')}")
+            lines.append(f"  Improvement: {comparison.get('improvement', 0):.1f}%")
+
+        return "\n".join(lines)
+
+    def _default_evaluation(self, response: str) -> float:
+        """Default quality evaluation (length-based)."""
+        # Simple heuristic: penalize very short or very long responses
+        length = len(response)
+        if length < 50:
+            return 50.0
+        elif length > 2000:
+            return 70.0
+        else:
+            return 85.0
+
+    def export_results(self, filepath: str) -> None:
+        """Export results to JSON file."""
+        data = {
+            "name": self.name,
+            "variants": {k: asdict(v) for k, v in self.variants.items()},
+            "results": [asdict(r) for r in self.results],
+            "summary": {
+                variant_id: self.calculate_metrics(variant_id)
+                for variant_id in self.variants.keys()
+            }
+        }
+
+        with open(filepath, 'w') as f:
+            json.dump(data, f, indent=2)
+
+        print(f"✅ Results exported to {filepath}")
+
+
+class ABTestRunner:
+    """Runner for executing A/B tests with actual LLM calls."""
+
+    def __init__(self, llm_client: Optional[Any] = None):
+        self.llm_client = llm_client
+
+    async def run_test(
+        self,
+        test: ABTest,
+        test_cases: List[Dict[str, Any]],
+        samples_per_variant: int = 30
+    ) -> ABTest:
+        """
+        Run an A/B test with multiple test cases.
+
+        Args:
+            test: The ABTest instance
+            test_cases: List of test case dictionaries with template variables
+            samples_per_variant: Number of samples to collect per variant
+
+        Returns:
+            The test instance with results
+        """
+        for variant_id, variant in test.variants.items():
+            print(f"Testing variant: {variant.name} ({samples_per_variant} samples)")
+
+            for i, test_case in enumerate(test_cases[:samples_per_variant]):
+                # Render prompt with test case
+                # In production, use actual template rendering
+                prompt = variant.template.format(**test_case)
+
+                # Simulate LLM call (replace with actual API call)
+                result = await self._simulate_llm_call(prompt, variant_id)
+
+                # Evaluate quality
+                quality_score = test.evaluation_fn(result.response)
+                result.quality_score = quality_score
+
+                test.add_result(result)
+
+                if (i + 1) % 10 == 0:
+                    print(f"  Progress: {i + 1}/{samples_per_variant}")
+
+        return test
+
+    async def _simulate_llm_call(
+        self,
+        prompt: str,
+        variant_id: str
+    ) -> TestResult:
+        """Simulate an LLM API call (replace with actual implementation)."""
+        # In production, call actual LLM API here
+        await asyncio.sleep(0.1)  # Simulate API latency
+
+        # Generate mock response
+        response = f"This is a simulated response for variant {variant_id}. " + "Sample content. " * 20
+
+        return TestResult(
+            variant_id=variant_id,
+            response=response,
+            latency_ms=np.random.uniform(200, 800),
+            tokens_used=np.random.randint(100, 500),
+            cost=np.random.uniform(0.001, 0.01)
+        )
+
+
+# Example usage
+if __name__ == "__main__":
+    # Define variants
+    variant_a = PromptVariant(
+        id="v1_concise",
+        name="Concise Prompt",
+        template="""Analyze this marketing campaign: {campaign_details}
+
+Provide brief recommendations.""",
+        metadata={"author": "Team A", "hypothesis": "Shorter prompts = faster responses"}
+    )
+
+    variant_b = PromptVariant(
+        id="v2_detailed",
+        name="Detailed Prompt",
+        template="""You are a marketing analytics expert. Analyze the following campaign in detail.
+
+Campaign Details: {campaign_details}
+
+Please provide:
+1. Performance assessment
+2. Key insights
+3. Specific recommendations
+4. Action items
+
+Be thorough and data-driven.""",
+        metadata={"author": "Team B", "hypothesis": "Detailed prompts = better quality"}
+    )
+
+    # Create test
+    test = ABTest(
+        name="Marketing Prompt Optimization Q1 2025",
+        variants=[variant_a, variant_b]
+    )
+
+    # Define test cases
+    test_cases = [
+        {
+            "campaign_details": f"Campaign {i}: Budget $10K, CTR 2.5%, Conv 1.2%"
+        }
+        for i in range(50)
+    ]
+
+    # Run test
+    runner = ABTestRunner()
+
+    async def run():
+        test_with_results = await runner.run_test(
+            test=test,
+            test_cases=test_cases,
+            samples_per_variant=25
+        )
+
+        # Generate report
+        print("\n" + test_with_results.generate_report())
+
+        # Compare variants
+        print("\n" + "=" * 80)
+        comparison = test_with_results.compare_variants("v1_concise", "v2_detailed", "quality_score")
+        print(f"\n🏆 Winner: {comparison['winner']}")
+        print(f"📊 Statistical Significance: {'Yes ✓' if comparison['significant'] else 'No ✗'}")
+        print(f"📈 Improvement: {comparison['improvement']:.1f}%")
+
+        # Export results
+        test_with_results.export_results("ab_test_results.json")
+
+    # Run async test
+    asyncio.run(run())

package/.claude/roles/ai-engineer/skills/01-prompt-engineering/prompt_template_manager.py

@@ -0,0 +1,274 @@
+"""
+Prompt Template Manager with Version Control
+Manages prompt templates with variable injection, inheritance, and versioning.
+"""
+
+import json
+import os
+from datetime import datetime
+from pathlib import Path
+from typing import Dict, Any, Optional, List
+from jinja2 import Environment, FileSystemLoader, Template
+import yaml
+
+
+class PromptTemplate:
+    """Version-controlled prompt template with variable injection."""
+
+    def __init__(
+        self,
+        name: str,
+        template: str,
+        version: str = "1.0.0",
+        metadata: Optional[Dict[str, Any]] = None,
+        parent: Optional[str] = None
+    ):
+        self.name = name
+        self.template = template
+        self.version = version
+        self.metadata = metadata or {}
+        self.parent = parent
+        self.created_at = datetime.now().isoformat()
+        self._jinja_template = Template(template)
+
+    def render(self, **kwargs) -> str:
+        """Render the template with provided variables."""
+        return self._jinja_template.render(**kwargs)
+
+    def to_dict(self) -> Dict[str, Any]:
+        """Convert to dictionary for serialization."""
+        return {
+            "name": self.name,
+            "template": self.template,
+            "version": self.version,
+            "metadata": self.metadata,
+            "parent": self.parent,
+            "created_at": self.created_at
+        }
+
+    @classmethod
+    def from_dict(cls, data: Dict[str, Any]) -> "PromptTemplate":
+        """Create from dictionary."""
+        return cls(
+            name=data["name"],
+            template=data["template"],
+            version=data.get("version", "1.0.0"),
+            metadata=data.get("metadata", {}),
+            parent=data.get("parent")
+        )
+
+    def save(self, directory: str = "./prompts") -> None:
+        """Save template to disk."""
+        Path(directory).mkdir(parents=True, exist_ok=True)
+        filepath = Path(directory) / f"{self.name}_v{self.version}.json"
+
+        with open(filepath, 'w') as f:
+            json.dump(self.to_dict(), f, indent=2)
+
+        print(f"✅ Saved: {filepath}")
+
+    @classmethod
+    def load(cls, name: str, version: Optional[str] = None, directory: str = "./prompts") -> "PromptTemplate":
+        """Load template from disk."""
+        if version:
+            filepath = Path(directory) / f"{name}_v{version}.json"
+        else:
+            # Load latest version
+            pattern = f"{name}_v*.json"
+            files = sorted(Path(directory).glob(pattern), reverse=True)
+            if not files:
+                raise FileNotFoundError(f"No template found for {name}")
+            filepath = files[0]
+
+        with open(filepath, 'r') as f:
+            data = json.load(f)
+
+        return cls.from_dict(data)
+
+
+class PromptLibrary:
+    """Centralized library for managing multiple prompt templates."""
+
+    def __init__(self, library_path: str = "./prompt_library"):
+        self.library_path = Path(library_path)
+        self.library_path.mkdir(parents=True, exist_ok=True)
+        self.templates: Dict[str, PromptTemplate] = {}
+        self._load_all()
+
+    def _load_all(self) -> None:
+        """Load all templates from library."""
+        for filepath in self.library_path.glob("*.json"):
+            with open(filepath, 'r') as f:
+                data = json.load(f)
+                template = PromptTemplate.from_dict(data)
+                self.templates[template.name] = template
+
+    def add(self, template: PromptTemplate) -> None:
+        """Add a template to the library."""
+        self.templates[template.name] = template
+        template.save(str(self.library_path))
+
+    def get(self, name: str, version: Optional[str] = None) -> PromptTemplate:
+        """Get a template by name."""
+        if version:
+            return PromptTemplate.load(name, version, str(self.library_path))
+        return self.templates.get(name) or PromptTemplate.load(name, directory=str(self.library_path))
+
+    def list(self) -> List[Dict[str, Any]]:
+        """List all templates."""
+        return [
+            {
+                "name": t.name,
+                "version": t.version,
+                "created_at": t.created_at,
+                "parent": t.parent
+            }
+            for t in self.templates.values()
+        ]
+
+    def create_from_yaml(self, yaml_path: str) -> PromptTemplate:
+        """Create template from YAML configuration."""
+        with open(yaml_path, 'r') as f:
+            config = yaml.safe_load(f)
+
+        template = PromptTemplate(
+            name=config["name"],
+            template=config["template"],
+            version=config.get("version", "1.0.0"),
+            metadata=config.get("metadata", {}),
+            parent=config.get("parent")
+        )
+
+        self.add(template)
+        return template
+
+
+# Example templates
+EXAMPLE_TEMPLATES = {
+    "marketing_email": """You are a marketing copywriter for {company}.
+
+Write a compelling email for {product} targeting {audience}.
+
+Requirements:
+- Tone: {tone}
+- Length: {length} words
+- Include a clear call-to-action
+- Use the brand voice: {brand_voice}
+
+Product Details:
+{product_details}
+
+Email:""",
+
+    "seo_optimizer": """Analyze and optimize the following content for SEO.
+
+Target Keywords: {keywords}
+Target Audience: {audience}
+
+Original Content:
+{content}
+
+Provide:
+1. Keyword density analysis
+2. Readability score
+3. Recommended improvements
+4. Optimized version
+
+Analysis:""",
+
+    "lead_scorer": """You are a lead scoring expert for B2B SaaS.
+
+Evaluate this lead based on the following criteria:
+
+Lead Information:
+- Company: {company}
+- Industry: {industry}
+- Company Size: {company_size}
+- Job Title: {job_title}
+- Engagement Level: {engagement_level}
+
+Scoring Criteria (0-100):
+1. Fit Score (company size, industry match)
+2. Interest Score (engagement, intent signals)
+3. Urgency Score (buying timeline indicators)
+
+Provide:
+- Overall Score (0-100)
+- Category: Hot/Warm/Cold
+- Recommended Action
+- Reasoning
+
+Analysis:"""
+}
+
+
+def create_default_library() -> PromptLibrary:
+    """Create a library with example templates."""
+    library = PromptLibrary()
+
+    for name, template_text in EXAMPLE_TEMPLATES.items():
+        template = PromptTemplate(
+            name=name,
+            template=template_text,
+            version="1.0.0",
+            metadata={
+                "author": "Tech Innovation Hub",
+                "category": "marketing",
+                "tags": ["production", "tested"]
+            }
+        )
+        library.add(template)
+
+    return library
+
+
+# Example usage
+if __name__ == "__main__":
+    # Create library with examples
+    library = create_default_library()
+
+    # List all templates
+    print("📚 Available Templates:")
+    for t in library.list():
+        print(f"  - {t['name']} (v{t['version']})")
+
+    print("\n" + "="*60 + "\n")
+
+    # Use marketing email template
+    template = library.get("marketing_email")
+
+    email = template.render(
+        company="Tech Innovation Hub",
+        product="AI Engineering Masterclass",
+        audience="Data Scientists and ML Engineers",
+        tone="professional yet approachable",
+        length="150",
+        brand_voice="innovative, data-driven, practical",
+        product_details="A comprehensive course covering LLMs, RAG, and Multi-Agent Systems"
+    )
+
+    print("📧 Generated Email:")
+    print(email)
+
+    print("\n" + "="*60 + "\n")
+
+    # Create a custom template
+    custom_template = PromptTemplate(
+        name="campaign_analyzer",
+        template="""Analyze this marketing campaign performance:
+
+Campaign: {campaign_name}
+Duration: {duration}
+Metrics:
+- Impressions: {impressions}
+- Clicks: {clicks}
+- Conversions: {conversions}
+- Revenue: ${revenue}
+
+Provide insights and recommendations.""",
+        version="1.0.0",
+        metadata={"author": "Marketing Team", "category": "analytics"}
+    )
+
+    library.add(custom_template)
+    print(f"✅ Added custom template: {custom_template.name}")