aipop 0.7.0__py3-none-any.whl

aipop/__init__.py

@@ -0,0 +1,6 @@
+"""AI Purple Ops core package."""
+
+from __future__ import annotations
+
+__all__ = ["__version__"]
+__version__ = "0.6.5"

aipop/adapters/__init__.py

@@ -0,0 +1,35 @@
+"""Model adapters for test execution."""
+
+from __future__ import annotations
+
+from .anthropic import AnthropicAdapter
+from .bedrock import BedrockAdapter
+from .huggingface import HuggingFaceAdapter
+from .llamacpp import LlamaCppAdapter
+from .mcp_adapter import MCPAdapter
+from .mock import MockAdapter
+from .ollama import OllamaAdapter
+from .openai import OpenAIAdapter
+from .registry import AdapterRegistry
+
+# Auto-register built-in adapters
+AdapterRegistry.register("mock", MockAdapter)
+AdapterRegistry.register("openai", OpenAIAdapter)
+AdapterRegistry.register("anthropic", AnthropicAdapter)
+AdapterRegistry.register("bedrock", BedrockAdapter)
+AdapterRegistry.register("huggingface", HuggingFaceAdapter)
+AdapterRegistry.register("ollama", OllamaAdapter)
+AdapterRegistry.register("llamacpp", LlamaCppAdapter)
+AdapterRegistry.register("mcp", MCPAdapter)
+
+__all__ = [
+    "AdapterRegistry",
+    "AnthropicAdapter",
+    "BedrockAdapter",
+    "HuggingFaceAdapter",
+    "LlamaCppAdapter",
+    "MCPAdapter",
+    "MockAdapter",
+    "OllamaAdapter",
+    "OpenAIAdapter",
+]

aipop/adapters/anthropic.py

@@ -0,0 +1,228 @@
+"""Anthropic Claude adapter with robust error handling."""
+
+from __future__ import annotations
+
+import os
+import time
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from typing import Any
+
+from tenacity import (
+    retry,
+    retry_if_exception_type,
+    stop_after_attempt,
+    wait_exponential,
+)
+
+from aipop.core.models import ModelResponse
+from aipop.utils.rate_limiter import RateLimiter
+
+
+class AnthropicAdapter:
+    """Anthropic Claude API adapter."""
+
+    def __init__(
+        self,
+        api_key: str | None = None,
+        model: str = "claude-3-opus-20240229",
+        timeout: int = 30,
+        max_retries: int = 3,
+        rpm_limit: int = 50,
+        proxy: str | None = None,
+    ) -> None:
+        """Initialize Anthropic adapter.
+
+        Args:
+            api_key: Anthropic API key (defaults to ANTHROPIC_API_KEY env var)
+            model: Model name (claude-3-opus, claude-3-sonnet, etc.)
+            timeout: Request timeout in seconds
+            max_retries: Maximum retry attempts
+            rpm_limit: Requests per minute limit (default: 50)
+            proxy: HTTP/SOCKS5 proxy URL (e.g., http://127.0.0.1:8080)
+        """
+        self.api_key = api_key or os.getenv("ANTHROPIC_API_KEY")
+        if not self.api_key:
+            raise ValueError(
+                "Anthropic API key not found. Set ANTHROPIC_API_KEY environment variable.\n"
+                "Example: export ANTHROPIC_API_KEY=sk-ant-...\n"
+                "Or use --adapter mock for testing without API keys."
+            )
+
+        self.model = model
+        self.timeout = timeout
+        self.max_retries = max_retries
+        self.proxy = proxy or os.getenv("HTTPS_PROXY") or os.getenv("HTTP_PROXY")
+        self.rate_limiter = RateLimiter(rpm=rpm_limit)
+
+        try:
+            from anthropic import Anthropic
+
+            # Route through proxy if configured
+            http_client = None
+            if self.proxy:
+                try:
+                    import httpx
+                    http_client = httpx.Client(proxy=self.proxy, verify=False)
+                except ImportError:
+                    pass
+
+            self.client = Anthropic(
+                api_key=self.api_key,
+                timeout=self.timeout,
+                http_client=http_client,
+            )
+        except ImportError as e:
+            msg = "Anthropic SDK not installed. Install with: pip install anthropic"
+            raise ImportError(msg) from e
+
+    @retry(
+        retry=retry_if_exception_type((Exception,)),
+        stop=stop_after_attempt(3),
+        wait=wait_exponential(multiplier=1, min=1, max=4),
+    )
+    def invoke(self, prompt: str, **kwargs: Any) -> ModelResponse:  # noqa: ANN401
+        """Invoke Claude model with prompt.
+
+        Args:
+            prompt: Input prompt
+            **kwargs: Additional parameters (temperature, max_tokens, etc.)
+
+        Returns:
+            ModelResponse with text and metadata
+        """
+        # Rate limit before API call
+        self.rate_limiter.acquire()
+
+        start_time = time.time()
+
+        try:
+            response = self.client.messages.create(
+                model=self.model,
+                max_tokens=kwargs.get("max_tokens", 1024),
+                messages=[{"role": "user", "content": prompt}],
+            )
+
+            latency_ms = (time.time() - start_time) * 1000
+
+            # Extract text content
+            text = ""
+            tool_calls = None
+
+            if response.content:
+                # Extract text from text blocks
+                text_blocks = [
+                    block.text
+                    for block in response.content
+                    if hasattr(block, "type") and block.type == "text"
+                ]
+                text = "".join(text_blocks)
+
+                # Extract tool_use blocks
+                tool_blocks = [
+                    block
+                    for block in response.content
+                    if hasattr(block, "type") and block.type == "tool_use"
+                ]
+                if tool_blocks:
+                    tool_calls = [
+                        {
+                            "id": block.id,
+                            "name": block.name,
+                            "arguments": block.input if hasattr(block, "input") else {},
+                        }
+                        for block in tool_blocks
+                    ]
+
+            # Calculate cost (approximate)
+            cost = self._calculate_cost(
+                response.usage.input_tokens,
+                response.usage.output_tokens,
+            )
+
+            return ModelResponse(
+                text=text,
+                tool_calls=tool_calls,
+                meta={
+                    "model": self.model,
+                    "latency_ms": latency_ms,
+                    "tokens_prompt": response.usage.input_tokens,
+                    "tokens_completion": response.usage.output_tokens,
+                    "cost_usd": cost,
+                    "finish_reason": response.stop_reason,
+                },
+            )
+
+        except Exception as e:
+            latency_ms = (time.time() - start_time) * 1000
+            raise RuntimeError(f"Anthropic API error: {e}") from e
+
+    def batch_query(self, prompts: list[str], **kwargs: Any) -> list[ModelResponse]:  # noqa: ANN401
+        """Execute batch of prompts in parallel using ThreadPoolExecutor.
+
+        Args:
+            prompts: List of prompts to execute
+            **kwargs: Additional parameters passed to invoke()
+
+        Returns:
+            List of ModelResponse objects in same order as prompts
+        """
+        # Use ThreadPoolExecutor for parallel execution
+        # Limit to 10 concurrent requests to avoid overwhelming the API
+        max_workers = min(10, len(prompts))
+
+        results = [None] * len(prompts)
+
+        with ThreadPoolExecutor(max_workers=max_workers) as executor:
+            # Submit all tasks
+            future_to_index = {
+                executor.submit(self.invoke, prompt, **kwargs): i
+                for i, prompt in enumerate(prompts)
+            }
+
+            # Collect results as they complete
+            for future in as_completed(future_to_index):
+                index = future_to_index[future]
+                try:
+                    results[index] = future.result()
+                except Exception as e:
+                    # Create error response for failed requests
+                    results[index] = ModelResponse(
+                        text="",
+                        meta={
+                            "error": str(e),
+                            "model": self.model,
+                            "latency_ms": 0,
+                            "cost_usd": 0,
+                        },
+                    )
+
+        return results
+
+    def _calculate_cost(self, input_tokens: int, output_tokens: int) -> float:
+        """Calculate approximate cost in USD.
+
+        Args:
+            input_tokens: Input tokens
+            output_tokens: Output tokens
+
+        Returns:
+            Cost in USD
+        """
+        # Pricing as of 2025-11-13 (approximate)
+        pricing = {
+            "claude-3-opus": {"input": 0.015 / 1000, "output": 0.075 / 1000},
+            "claude-3-sonnet": {"input": 0.003 / 1000, "output": 0.015 / 1000},
+            "claude-3-haiku": {"input": 0.00025 / 1000, "output": 0.00125 / 1000},
+        }
+
+        model_parts = self.model.split("-")[0:3]  # Extract parts: ["claude", "3", "opus"]
+        model_key = "-".join(model_parts)  # Join to "claude-3-opus"
+        if model_key not in pricing:
+            model_key = "claude-3-sonnet"  # Default
+
+        cost = (
+            input_tokens * pricing[model_key]["input"]
+            + output_tokens * pricing[model_key]["output"]
+        )
+
+        return cost

aipop/adapters/auto_probe.py

@@ -0,0 +1,343 @@
+"""Auto-probe adapter — discovers how to talk to a target from its URL.
+
+The zero-friction path: `aipop scan --target http://localhost:8000/chat`
+No YAML. No config. AIPOP figures out the request/response format.
+
+Probe strategy:
+1. Check for OpenAPI/Swagger spec at common paths
+2. Send probe requests with common field names
+3. Use whatever works
+
+If auto-probe fails, tell the user exactly what to specify.
+"""
+
+from __future__ import annotations
+
+import time
+from typing import Any
+
+import requests
+
+from aipop.adapters.custom_http import CustomHTTPAdapter
+from aipop.core.models import ModelResponse
+
+
+# Common prompt field names across AI APIs
+PROMPT_FIELDS = ["message", "prompt", "query", "input", "text", "content", "question"]
+
+# Common response field names across AI APIs
+RESPONSE_FIELDS = ["response", "reply", "output", "text", "answer", "content", "result",
+                    "choices.0.message.content", "data.response", "generated_text"]
+
+# Common OpenAPI/docs paths
+OPENAPI_PATHS = ["/openapi.json", "/docs", "/swagger.json", "/api-docs",
+                 "/.well-known/openapi.json"]
+
+PROBE_MESSAGE = "Hello, can you help me?"
+
+
+def probe_target(target_url: str, timeout: int = 15) -> dict[str, Any]:
+    """Probe a target URL and discover its request/response format.
+
+    Args:
+        target_url: The endpoint URL to probe
+        timeout: Request timeout in seconds
+
+    Returns:
+        Config dict compatible with CustomHTTPAdapter
+
+    Raises:
+        ProbeError: If the target can't be reached or format can't be determined
+    """
+    results = {
+        "base_url": target_url,
+        "method": "POST",
+        "prompt_field": None,
+        "response_field": None,
+        "extra_fields": {},
+        "openapi": None,
+        "probe_log": [],
+    }
+
+    # Step 1: Try to find OpenAPI spec
+    base = _extract_base_url(target_url)
+    for path in OPENAPI_PATHS:
+        try:
+            r = requests.get(f"{base}{path}", timeout=5)
+            if r.status_code == 200 and "paths" in r.text:
+                results["openapi"] = r.json()
+                results["probe_log"].append(f"Found OpenAPI spec at {base}{path}")
+                _extract_from_openapi(results, target_url)
+                if results["prompt_field"] and results["response_field"]:
+                    return results
+                break
+        except Exception:
+            continue
+
+    # Step 2: Probe with common field names
+    for prompt_field in PROMPT_FIELDS:
+        body = {prompt_field: PROBE_MESSAGE}
+
+        try:
+            r = requests.post(target_url, json=body, timeout=timeout,
+                              headers={"Content-Type": "application/json"})
+        except requests.ConnectionError:
+            raise ProbeError(
+                f"Can't connect to {target_url}. Is the target running?"
+            )
+        except requests.Timeout:
+            raise ProbeError(
+                f"Target at {target_url} timed out after {timeout}s."
+            )
+
+        if r.status_code >= 500:
+            results["probe_log"].append(
+                f"  {prompt_field} → {r.status_code} (server error)")
+            continue
+
+        if r.status_code == 422:
+            # Validation error — wrong field name, try next
+            results["probe_log"].append(
+                f"  {prompt_field} → 422 (validation error, wrong field)")
+            continue
+
+        if r.status_code == 405:
+            # Method not allowed — might be GET
+            results["probe_log"].append(
+                f"  POST → 405 (method not allowed)")
+            break
+
+        if r.status_code in (200, 201):
+            results["prompt_field"] = prompt_field
+            results["probe_log"].append(
+                f"  {prompt_field} → {r.status_code} (accepted!)")
+
+            # Now find the response field
+            try:
+                data = r.json()
+                for response_field in RESPONSE_FIELDS:
+                    value = _extract_nested(data, response_field)
+                    if value is not None and isinstance(value, str) and len(value) > 5:
+                        results["response_field"] = response_field
+                        results["probe_log"].append(
+                            f"  Response field: {response_field}")
+                        return results
+
+                # Didn't find a known field — list what's available
+                available = _list_string_fields(data)
+                if available:
+                    # Pick the first string field that looks like a response
+                    best = _pick_best_response_field(available, data)
+                    if best:
+                        results["response_field"] = best
+                        results["probe_log"].append(
+                            f"  Response field (guessed): {best}")
+                        return results
+
+                results["probe_log"].append(
+                    f"  Got response but can't find text field. "
+                    f"Available: {available}")
+            except ValueError:
+                results["probe_log"].append(
+                    f"  Got {r.status_code} but response isn't JSON")
+                continue
+
+    if results["prompt_field"] and not results["response_field"]:
+        raise ProbeError(
+            f"Target accepts requests with field '{results['prompt_field']}' "
+            f"but the response format is unknown.\n"
+            f"Use --response-field to specify which JSON field contains the text.\n"
+            f"Probe log:\n" + "\n".join(results["probe_log"])
+        )
+
+    if not results["prompt_field"]:
+        raise ProbeError(
+            f"Can't figure out how to talk to {target_url}.\n"
+            f"Tried prompt fields: {', '.join(PROMPT_FIELDS)}\n"
+            f"Use --prompt-field and --response-field to specify manually.\n"
+            f"Or create an adapter YAML: cp templates/adapters/custom_http.yaml "
+            f"adapters/my_target.yaml\n"
+            f"Probe log:\n" + "\n".join(results["probe_log"])
+        )
+
+    return results
+
+
+def build_adapter_from_probe(
+    target_url: str,
+    prompt_field: str | None = None,
+    response_field: str | None = None,
+    timeout: int = 15,
+) -> CustomHTTPAdapter:
+    """Build a working adapter for a target URL.
+
+    Auto-probes if field names aren't specified. Uses explicit values if given.
+
+    Args:
+        target_url: The endpoint URL
+        prompt_field: Override prompt field name (skip probe for request format)
+        response_field: Override response field name (skip probe for response format)
+        timeout: Request timeout
+
+    Returns:
+        Ready-to-use CustomHTTPAdapter
+    """
+    if prompt_field and response_field:
+        # User told us everything — no probe needed
+        config = {
+            "connection": {
+                "base_url": target_url,
+                "method": "POST",
+                "timeout": timeout,
+                "headers": {"Content-Type": "application/json"},
+            },
+            "auth": {"type": "none"},
+            "request": {"prompt_field": prompt_field},
+            "response": {"text_field": response_field},
+        }
+        return CustomHTTPAdapter(config)
+
+    # Probe the target
+    probe_result = probe_target(target_url, timeout=timeout)
+
+    # Override with explicit values if given
+    if prompt_field:
+        probe_result["prompt_field"] = prompt_field
+    if response_field:
+        probe_result["response_field"] = response_field
+
+    config = {
+        "connection": {
+            "base_url": probe_result["base_url"],
+            "method": probe_result["method"],
+            "timeout": timeout,
+            "headers": {"Content-Type": "application/json"},
+        },
+        "auth": {"type": "none"},
+        "request": {
+            "prompt_field": probe_result["prompt_field"],
+            "extra_fields": probe_result.get("extra_fields", {}),
+        },
+        "response": {"text_field": probe_result["response_field"]},
+    }
+    return CustomHTTPAdapter(config)
+
+
+def _extract_base_url(url: str) -> str:
+    """Extract scheme + host from a full URL."""
+    from urllib.parse import urlparse
+    parsed = urlparse(url)
+    return f"{parsed.scheme}://{parsed.netloc}"
+
+
+def _extract_nested(data: dict, path: str) -> Any:
+    """Extract a value from a nested dict using dot notation."""
+    current = data
+    for part in path.split("."):
+        if isinstance(current, dict) and part in current:
+            current = current[part]
+        elif isinstance(current, list):
+            try:
+                current = current[int(part)]
+            except (ValueError, IndexError):
+                return None
+        else:
+            return None
+    return current
+
+
+def _list_string_fields(data: dict, prefix: str = "") -> list[str]:
+    """List all fields that contain strings in a JSON response."""
+    fields = []
+    if not isinstance(data, dict):
+        return fields
+    for key, value in data.items():
+        path = f"{prefix}.{key}" if prefix else key
+        if isinstance(value, str):
+            fields.append(path)
+        elif isinstance(value, dict):
+            fields.extend(_list_string_fields(value, path))
+    return fields
+
+
+def _pick_best_response_field(fields: list[str], data: dict) -> str | None:
+    """Pick the most likely response field from available string fields.
+
+    Heuristic: longest string value that isn't a field like 'id', 'status', 'model'.
+    """
+    skip_patterns = {"id", "status", "model", "type", "name", "version",
+                     "created", "object", "app", "error"}
+
+    candidates = []
+    for field in fields:
+        leaf = field.split(".")[-1].lower()
+        if leaf in skip_patterns:
+            continue
+        value = _extract_nested(data, field)
+        if isinstance(value, str) and len(value) > 10:
+            candidates.append((field, len(value)))
+
+    if not candidates:
+        return None
+
+    # Return the field with the longest value — most likely the actual response
+    candidates.sort(key=lambda x: x[1], reverse=True)
+    return candidates[0][0]
+
+
+def _extract_from_openapi(results: dict, target_url: str) -> None:
+    """Try to extract prompt/response fields from an OpenAPI spec."""
+    spec = results.get("openapi", {})
+    if not spec:
+        return
+
+    # Find the path that matches our target URL
+    from urllib.parse import urlparse
+    parsed = urlparse(target_url)
+    target_path = parsed.path
+
+    paths = spec.get("paths", {})
+    if target_path not in paths:
+        return
+
+    post = paths[target_path].get("post", {})
+    if not post:
+        return
+
+    # Check request body schema
+    body = post.get("requestBody", {})
+    schema_ref = (body.get("content", {})
+                  .get("application/json", {})
+                  .get("schema", {}))
+
+    # Resolve $ref
+    if "$ref" in schema_ref:
+        ref_name = schema_ref["$ref"].split("/")[-1]
+        schema = spec.get("components", {}).get("schemas", {}).get(ref_name, {})
+    else:
+        schema = schema_ref
+
+    props = schema.get("properties", {})
+    required = schema.get("required", [])
+
+    # Find the prompt field — required string field, or common names
+    for field_name in PROMPT_FIELDS:
+        if field_name in props:
+            results["prompt_field"] = field_name
+            results["probe_log"].append(
+                f"  OpenAPI: prompt field = {field_name}")
+            break
+
+    if not results["prompt_field"] and required:
+        # Use the first required string field
+        for req in required:
+            if req in props and props[req].get("type") == "string":
+                results["prompt_field"] = req
+                results["probe_log"].append(
+                    f"  OpenAPI: prompt field (from required) = {req}")
+                break
+
+
+class ProbeError(Exception):
+    """Raised when auto-probe can't determine how to talk to the target."""
+    pass