promptum 0.0.1__py3-none-any.whl

promptum/__init__.py

@@ -0,0 +1,44 @@
+from promptum.benchmark import Benchmark, Report
+from promptum.core import Metrics, RetryConfig, RetryStrategy, TestCase, TestResult
+from promptum.execution import Runner
+from promptum.providers import LLMProvider, OpenRouterClient
+from promptum.serialization import (
+    HTMLSerializer,
+    JSONSerializer,
+    Serializer,
+    YAMLSerializer,
+)
+from promptum.storage import FileStorage, ResultStorage
+from promptum.validation import (
+    Contains,
+    ExactMatch,
+    JsonSchema,
+    Regex,
+    Validator,
+)
+
+__version__ = "0.1.0"
+
+__all__ = [
+    "TestCase",
+    "TestResult",
+    "Metrics",
+    "RetryConfig",
+    "RetryStrategy",
+    "Validator",
+    "ExactMatch",
+    "Contains",
+    "Regex",
+    "JsonSchema",
+    "LLMProvider",
+    "OpenRouterClient",
+    "Runner",
+    "Benchmark",
+    "Report",
+    "Serializer",
+    "JSONSerializer",
+    "YAMLSerializer",
+    "HTMLSerializer",
+    "ResultStorage",
+    "FileStorage",
+]

promptum/benchmark/__init__.py

@@ -0,0 +1,4 @@
+from promptum.benchmark.benchmark import Benchmark
+from promptum.benchmark.report import Report
+
+__all__ = ["Benchmark", "Report"]

promptum/benchmark/benchmark.py

@@ -0,0 +1,50 @@
+import asyncio
+from collections.abc import Callable, Sequence
+from typing import Any
+
+from promptum.benchmark.report import Report
+from promptum.core.result import TestResult
+from promptum.core.test_case import TestCase
+from promptum.execution.runner import Runner
+from promptum.providers.protocol import LLMProvider
+
+
+class Benchmark:
+    def __init__(
+        self,
+        provider: LLMProvider,
+        name: str = "benchmark",
+        max_concurrent: int = 5,
+        progress_callback: Callable[[int, int, TestResult], None] | None = None,
+    ):
+        self.provider = provider
+        self.name = name
+        self.max_concurrent = max_concurrent
+        self.progress_callback = progress_callback
+        self._test_cases: list[TestCase] = []
+
+    def add_test(self, test_case: TestCase) -> None:
+        self._test_cases.append(test_case)
+
+    def add_tests(self, test_cases: Sequence[TestCase]) -> None:
+        self._test_cases.extend(test_cases)
+
+    def run(self, metadata: dict[str, Any] | None = None) -> Report:
+        return asyncio.run(self.run_async(metadata))
+
+    async def run_async(self, metadata: dict[str, Any] | None = None) -> Report:
+        if not self._test_cases:
+            return Report(results=[], metadata=metadata or {})
+
+        runner = Runner(
+            provider=self.provider,
+            max_concurrent=self.max_concurrent,
+            progress_callback=self.progress_callback,
+        )
+
+        results = await runner.run(self._test_cases)
+
+        return Report(
+            results=results,
+            metadata=metadata or {},
+        )

promptum/benchmark/report.py

@@ -0,0 +1,75 @@
+from collections.abc import Callable, Sequence
+from dataclasses import dataclass
+from typing import Any
+
+from promptum.core.result import TestResult
+
+
+@dataclass(frozen=True, slots=True)
+class Report:
+    results: Sequence[TestResult]
+    metadata: dict[str, Any]
+
+    def get_summary(self) -> dict[str, Any]:
+        total = len(self.results)
+        passed = sum(1 for r in self.results if r.passed)
+
+        latencies = [r.metrics.latency_ms for r in self.results if r.metrics]
+        total_cost = sum(r.metrics.cost_usd or 0 for r in self.results if r.metrics)
+        total_tokens = sum(r.metrics.total_tokens or 0 for r in self.results if r.metrics)
+
+        return {
+            "total": total,
+            "passed": passed,
+            "failed": total - passed,
+            "pass_rate": passed / total if total > 0 else 0,
+            "avg_latency_ms": sum(latencies) / len(latencies) if latencies else 0,
+            "p50_latency_ms": self._percentile(latencies, 0.5) if latencies else 0,
+            "p95_latency_ms": self._percentile(latencies, 0.95) if latencies else 0,
+            "p99_latency_ms": self._percentile(latencies, 0.99) if latencies else 0,
+            "total_cost_usd": total_cost,
+            "total_tokens": total_tokens,
+        }
+
+    def filter(
+        self,
+        model: str | None = None,
+        tags: Sequence[str] | None = None,
+        passed: bool | None = None,
+    ) -> "Report":
+        filtered = list(self.results)
+
+        if model:
+            filtered = [r for r in filtered if r.test_case.model == model]
+
+        if tags:
+            tag_set = set(tags)
+            filtered = [r for r in filtered if tag_set.intersection(r.test_case.tags)]
+
+        if passed is not None:
+            filtered = [r for r in filtered if r.passed == passed]
+
+        return Report(results=filtered, metadata=self.metadata)
+
+    def group_by(self, key: Callable[[TestResult], str]) -> dict[str, "Report"]:
+        groups: dict[str, list[TestResult]] = {}
+
+        for result in self.results:
+            group_key = key(result)
+            if group_key not in groups:
+                groups[group_key] = []
+            groups[group_key].append(result)
+
+        return {k: Report(results=v, metadata=self.metadata) for k, v in groups.items()}
+
+    def compare_models(self) -> dict[str, dict[str, Any]]:
+        by_model = self.group_by(lambda r: r.test_case.model)
+        return {model: report.get_summary() for model, report in by_model.items()}
+
+    @staticmethod
+    def _percentile(values: list[float], p: float) -> float:
+        if not values:
+            return 0
+        sorted_values = sorted(values)
+        index = int(len(sorted_values) * p)
+        return sorted_values[min(index, len(sorted_values) - 1)]

promptum/core/__init__.py

@@ -0,0 +1,12 @@
+from promptum.core.metrics import Metrics
+from promptum.core.result import TestResult
+from promptum.core.retry import RetryConfig, RetryStrategy
+from promptum.core.test_case import TestCase
+
+__all__ = [
+    "Metrics",
+    "RetryConfig",
+    "RetryStrategy",
+    "TestCase",
+    "TestResult",
+]

promptum/core/metrics.py

@@ -0,0 +1,16 @@
+from collections.abc import Sequence
+from dataclasses import dataclass
+
+
+@dataclass(frozen=True, slots=True)
+class Metrics:
+    latency_ms: float
+    prompt_tokens: int | None = None
+    completion_tokens: int | None = None
+    total_tokens: int | None = None
+    cost_usd: float | None = None
+    retry_delays: Sequence[float] = ()
+
+    @property
+    def total_attempts(self) -> int:
+        return len(self.retry_delays) + 1

promptum/core/result.py

@@ -0,0 +1,17 @@
+from dataclasses import dataclass, field
+from datetime import datetime
+from typing import Any
+
+from promptum.core.metrics import Metrics
+from promptum.core.test_case import TestCase
+
+
+@dataclass(frozen=True, slots=True)
+class TestResult:
+    test_case: TestCase
+    response: str | None
+    passed: bool
+    metrics: Metrics | None
+    validation_details: dict[str, Any]
+    execution_error: str | None = None
+    timestamp: datetime = field(default_factory=datetime.now)

promptum/core/retry.py

@@ -0,0 +1,19 @@
+from collections.abc import Sequence
+from dataclasses import dataclass
+from enum import Enum
+
+
+class RetryStrategy(Enum):
+    EXPONENTIAL_BACKOFF = "exponential_backoff"
+    FIXED_DELAY = "fixed_delay"
+
+
+@dataclass(frozen=True, slots=True)
+class RetryConfig:
+    max_attempts: int = 3
+    strategy: RetryStrategy = RetryStrategy.EXPONENTIAL_BACKOFF
+    initial_delay: float = 1.0
+    max_delay: float = 60.0
+    exponential_base: float = 2.0
+    retryable_status_codes: Sequence[int] = (429, 500, 502, 503, 504)
+    timeout: float = 60.0

promptum/core/test_case.py

@@ -0,0 +1,22 @@
+from collections.abc import Sequence
+from dataclasses import dataclass, field
+from typing import TYPE_CHECKING, Any
+
+if TYPE_CHECKING:
+    from promptum.validation.protocol import Validator
+
+from promptum.core.retry import RetryConfig
+
+
+@dataclass(frozen=True, slots=True)
+class TestCase:
+    name: str
+    prompt: str
+    model: str
+    validator: "Validator"
+    tags: Sequence[str] = ()
+    system_prompt: str | None = None
+    temperature: float = 1.0
+    max_tokens: int | None = None
+    retry_config: RetryConfig | None = None
+    metadata: dict[str, Any] = field(default_factory=dict)

promptum/execution/__init__.py

@@ -0,0 +1,3 @@
+from promptum.execution.runner import Runner
+
+__all__ = ["Runner"]

promptum/execution/runner.py

@@ -0,0 +1,75 @@
+import asyncio
+from collections.abc import Callable, Sequence
+
+import httpx
+
+from promptum.core.result import TestResult
+from promptum.core.test_case import TestCase
+from promptum.providers.protocol import LLMProvider
+
+
+class Runner:
+    def __init__(
+        self,
+        provider: LLMProvider,
+        max_concurrent: int = 5,
+        progress_callback: Callable[[int, int, TestResult], None] | None = None,
+    ):
+        self.provider = provider
+        self.max_concurrent = max_concurrent
+        self.progress_callback = progress_callback
+
+    async def run(self, test_cases: Sequence[TestCase]) -> list[TestResult]:
+        semaphore = asyncio.Semaphore(self.max_concurrent)
+        completed = 0
+        total = len(test_cases)
+
+        async def run_with_semaphore(test_case: TestCase) -> TestResult:
+            async with semaphore:
+                result = await self._run_single_test(test_case)
+
+                nonlocal completed
+                completed += 1
+                if self.progress_callback:
+                    self.progress_callback(completed, total, result)
+
+                return result
+
+        results = await asyncio.gather(
+            *[run_with_semaphore(tc) for tc in test_cases],
+            return_exceptions=False,
+        )
+
+        return list(results)
+
+    async def _run_single_test(self, test_case: TestCase) -> TestResult:
+        try:
+            response, metrics = await self.provider.generate(
+                prompt=test_case.prompt,
+                model=test_case.model,
+                system_prompt=test_case.system_prompt,
+                temperature=test_case.temperature,
+                max_tokens=test_case.max_tokens,
+                retry_config=test_case.retry_config,
+            )
+
+            passed, validation_details = test_case.validator.validate(response)
+
+            return TestResult(
+                test_case=test_case,
+                response=response,
+                passed=passed,
+                metrics=metrics,
+                validation_details=validation_details,
+                execution_error=None,
+            )
+
+        except (RuntimeError, ValueError, TypeError, httpx.HTTPError) as e:
+            return TestResult(
+                test_case=test_case,
+                response=None,
+                passed=False,
+                metrics=None,
+                validation_details={},
+                execution_error=str(e),
+            )

promptum/providers/__init__.py

@@ -0,0 +1,7 @@
+from promptum.providers.openrouter import OpenRouterClient
+from promptum.providers.protocol import LLMProvider
+
+__all__ = [
+    "LLMProvider",
+    "OpenRouterClient",
+]

promptum/providers/openrouter.py

@@ -0,0 +1,123 @@
+import asyncio
+import time
+from typing import Any
+
+import httpx
+
+from promptum.core.metrics import Metrics
+from promptum.core.retry import RetryConfig, RetryStrategy
+
+
+class OpenRouterClient:
+    def __init__(
+        self,
+        api_key: str,
+        base_url: str = "https://openrouter.ai/api/v1",
+        default_retry_config: RetryConfig | None = None,
+    ):
+        self.api_key = api_key
+        self.base_url = base_url
+        self.default_retry_config = default_retry_config or RetryConfig()
+        self._client: httpx.AsyncClient | None = None
+
+    async def __aenter__(self) -> "OpenRouterClient":
+        self._client = httpx.AsyncClient(
+            base_url=self.base_url,
+            headers={
+                "Authorization": f"Bearer {self.api_key}",
+                "Content-Type": "application/json",
+            },
+            timeout=self.default_retry_config.timeout,
+        )
+        return self
+
+    async def __aexit__(self, *args: Any) -> None:
+        if self._client:
+            await self._client.aclose()
+
+    async def generate(
+        self,
+        prompt: str,
+        model: str,
+        system_prompt: str | None = None,
+        temperature: float = 1.0,
+        max_tokens: int | None = None,
+        retry_config: RetryConfig | None = None,
+        **kwargs: Any,
+    ) -> tuple[str, Metrics]:
+        if not self._client:
+            raise RuntimeError("Client not initialized. Use async context manager.")
+
+        config = retry_config or self.default_retry_config
+        retry_delays: list[float] = []
+
+        messages = []
+        if system_prompt:
+            messages.append({"role": "system", "content": system_prompt})
+        messages.append({"role": "user", "content": prompt})
+
+        payload: dict[str, Any] = {
+            "model": model,
+            "messages": messages,
+            "temperature": temperature,
+        }
+        if max_tokens:
+            payload["max_tokens"] = max_tokens
+        payload.update(kwargs)
+
+        for attempt in range(config.max_attempts):
+            start_time = time.perf_counter()
+            try:
+                response = await self._client.post(
+                    "/chat/completions",
+                    json=payload,
+                    timeout=config.timeout,
+                )
+
+                if response.status_code == 200:
+                    latency_ms = (time.perf_counter() - start_time) * 1000
+                    try:
+                        data = response.json()
+                        content = data["choices"][0]["message"]["content"]
+                    except (KeyError, IndexError, TypeError) as e:
+                        raise RuntimeError(f"Invalid API response structure: {e}") from e
+
+                    usage = data.get("usage", {})
+                    metrics = Metrics(
+                        latency_ms=latency_ms,
+                        prompt_tokens=usage.get("prompt_tokens"),
+                        completion_tokens=usage.get("completion_tokens"),
+                        total_tokens=usage.get("total_tokens"),
+                        cost_usd=usage.get("cost") or usage.get("total_cost"),
+                        retry_delays=tuple(retry_delays),
+                    )
+
+                    return content, metrics
+
+                if response.status_code not in config.retryable_status_codes:
+                    response.raise_for_status()
+
+                if attempt < config.max_attempts - 1:
+                    delay = self._calculate_delay(attempt, config)
+                    retry_delays.append(delay)
+                    await asyncio.sleep(delay)
+
+            except (httpx.TimeoutException, httpx.NetworkError) as e:
+                if attempt < config.max_attempts - 1:
+                    delay = self._calculate_delay(attempt, config)
+                    retry_delays.append(delay)
+                    await asyncio.sleep(delay)
+                else:
+                    raise RuntimeError(
+                        f"Request failed after {config.max_attempts} attempts: {e}"
+                    ) from e
+            except httpx.HTTPStatusError as e:
+                raise RuntimeError(f"HTTP error {e.response.status_code}: {e.response.text}") from e
+
+        raise RuntimeError(f"Request failed after {config.max_attempts} attempts")
+
+    def _calculate_delay(self, attempt: int, config: RetryConfig) -> float:
+        if config.strategy == RetryStrategy.EXPONENTIAL_BACKOFF:
+            delay = config.initial_delay * (config.exponential_base**attempt)
+            return min(delay, config.max_delay)
+        return config.initial_delay

promptum/providers/protocol.py

@@ -0,0 +1,22 @@
+from typing import Any, Protocol
+
+from promptum.core.metrics import Metrics
+
+
+class LLMProvider(Protocol):
+    async def generate(
+        self,
+        prompt: str,
+        model: str,
+        system_prompt: str | None = None,
+        temperature: float = 1.0,
+        max_tokens: int | None = None,
+        **kwargs: Any,
+    ) -> tuple[str, Metrics]:
+        """
+        Generates a response from the LLM.
+
+        Returns:
+            (response_text, metrics)
+        """
+        ...

promptum/serialization/__init__.py

@@ -0,0 +1,11 @@
+from promptum.serialization.html import HTMLSerializer
+from promptum.serialization.json import JSONSerializer
+from promptum.serialization.protocol import Serializer
+from promptum.serialization.yaml import YAMLSerializer
+
+__all__ = [
+    "Serializer",
+    "JSONSerializer",
+    "YAMLSerializer",
+    "HTMLSerializer",
+]

promptum/serialization/base.py

@@ -0,0 +1,48 @@
+"""Base serializer with shared result serialization logic."""
+
+from typing import Any
+
+from promptum.core.result import TestResult
+
+
+class BaseSerializer:
+    """
+    Base class for serializers with common result serialization logic.
+
+    Subclasses should implement:
+    - serialize(report: Report) -> str
+    - get_file_extension() -> str
+    """
+
+    @staticmethod
+    def _serialize_result(result: TestResult) -> dict[str, Any]:
+        """Convert TestResult to dictionary representation."""
+        return {
+            "test_case": {
+                "name": result.test_case.name,
+                "prompt": result.test_case.prompt,
+                "model": result.test_case.model,
+                "tags": list(result.test_case.tags),
+                "system_prompt": result.test_case.system_prompt,
+                "temperature": result.test_case.temperature,
+                "max_tokens": result.test_case.max_tokens,
+                "metadata": result.test_case.metadata,
+                "validator": result.test_case.validator.describe(),
+            },
+            "response": result.response,
+            "passed": result.passed,
+            "metrics": {
+                "latency_ms": result.metrics.latency_ms,
+                "prompt_tokens": result.metrics.prompt_tokens,
+                "completion_tokens": result.metrics.completion_tokens,
+                "total_tokens": result.metrics.total_tokens,
+                "cost_usd": result.metrics.cost_usd,
+                "retry_delays": list(result.metrics.retry_delays),
+                "total_attempts": result.metrics.total_attempts,
+            }
+            if result.metrics
+            else None,
+            "validation_details": result.validation_details,
+            "execution_error": result.execution_error,
+            "timestamp": result.timestamp.isoformat(),
+        }

promptum/serialization/html.py

@@ -0,0 +1,52 @@
+import json
+from pathlib import Path
+
+from jinja2 import Template
+
+from promptum.benchmark.report import Report
+
+
+class HTMLSerializer:
+    def __init__(self) -> None:
+        template_path = Path(__file__).parent / "report_template.html"
+        self._template = Template(template_path.read_text())
+
+    def serialize(self, report: Report) -> str:
+        summary = report.get_summary()
+
+        results_data = []
+        for result in report.results:
+            results_data.append(
+                {
+                    "test_case": {
+                        "name": result.test_case.name,
+                        "prompt": result.test_case.prompt,
+                        "model": result.test_case.model,
+                        "tags": list(result.test_case.tags),
+                        "system_prompt": result.test_case.system_prompt,
+                        "validator": result.test_case.validator.describe(),
+                    },
+                    "response": result.response,
+                    "passed": result.passed,
+                    "metrics": {
+                        "latency_ms": result.metrics.latency_ms,
+                        "prompt_tokens": result.metrics.prompt_tokens,
+                        "completion_tokens": result.metrics.completion_tokens,
+                        "total_tokens": result.metrics.total_tokens,
+                        "cost_usd": result.metrics.cost_usd,
+                        "total_attempts": result.metrics.total_attempts,
+                    }
+                    if result.metrics
+                    else None,
+                    "execution_error": result.execution_error,
+                }
+            )
+
+        return self._template.render(
+            summary=summary,
+            results=results_data,
+            results_json=json.dumps(results_data),
+        )
+
+    def get_file_extension(self) -> str:
+        return "html"

promptum/serialization/json.py

@@ -0,0 +1,28 @@
+import json
+from datetime import datetime
+from typing import Any
+
+from promptum.benchmark.report import Report
+from promptum.serialization.base import BaseSerializer
+
+
+class JSONSerializer(BaseSerializer):
+    def __init__(self, indent: int = 2):
+        self.indent = indent
+
+    def serialize(self, report: Report) -> str:
+        data = {
+            "metadata": report.metadata,
+            "summary": report.get_summary(),
+            "results": [self._serialize_result(r) for r in report.results],
+        }
+        return json.dumps(data, indent=self.indent, default=self._json_default)
+
+    def get_file_extension(self) -> str:
+        return "json"
+
+    @staticmethod
+    def _json_default(obj: Any) -> Any:
+        if isinstance(obj, datetime):
+            return obj.isoformat()
+        raise TypeError(f"Object of type {type(obj)} is not JSON serializable")

promptum/serialization/protocol.py

@@ -0,0 +1,13 @@
+from typing import Protocol
+
+from promptum.benchmark.report import Report
+
+
+class Serializer(Protocol):
+    def serialize(self, report: Report) -> str:
+        """Serializes a Report to a string format."""
+        ...
+
+    def get_file_extension(self) -> str:
+        """Returns the file extension for this format (e.g., 'json', 'html')."""
+        ...