openbench-ai 0.4.0__py3-none-any.whl

openbench/__init__.py

@@ -0,0 +1,191 @@
+"""OpenBench AI: A modular, open-source LLM evaluation, execution, and benchmarking platform."""
+
+from openbench.adapters import (
+    PROVIDER_REGISTRY,
+    BaseModelAdapter,
+    DeterministicLookupAdapter,
+    GenerationConfig,
+    HuggingFaceAdapter,
+    LocalCallableAdapter,
+    MockModelProvider,
+    OllamaAdapter,
+    OllamaConnectionError,
+    ProviderHealthStatus,
+    create_provider,
+    list_registered_providers,
+    register_provider,
+)
+from openbench.analytics import BenchmarkAnalytics
+from openbench.cache import CacheRecord, ResponseCache
+from openbench.catalog import DatasetCatalog, DatasetCatalogEntry
+from openbench.comparison import (
+    ComparisonResult,
+    MetricLeader,
+    ModelComparator,
+    ModelComparisonEntry,
+)
+from openbench.configuration import EvaluationProfile, ProfileLoader
+from openbench.dataset import (
+    Dataset,
+    DatasetLoader,
+    DatasetRecord,
+    IdFilterSpec,
+    load_ids_file,
+)
+from openbench.evaluator import (
+    EvaluationConfig,
+    EvaluationResult,
+    Evaluator,
+    ItemEvaluationResult,
+    ModelResponseSet,
+    ResponseItem,
+    ResponseLoader,
+)
+from openbench.exceptions import (
+    ConfigurationError,
+    DatasetValidationError,
+    EmptyDatasetError,
+    IncompatibleComparisonError,
+    MetricError,
+    OpenBenchError,
+    ResponseValidationError,
+)
+from openbench.execution import (
+    ExecutionConfig,
+    ExecutionEngine,
+    ExecutionOutcome,
+    RetryPolicy,
+    classify_execution_error,
+)
+from openbench.experiments import ExperimentManager, ExperimentRecord, RepeatGroupResult
+from openbench.history import HistoryEntry, ResultHistoryStore
+from openbench.integrity import (
+    IntegrityCheckResult,
+    RunIntegrityVerifier,
+    compute_dict_sha256,
+    compute_file_sha256,
+)
+from openbench.lineage import RunLineage
+from openbench.metrics import LexicalSimilarityMetric
+from openbench.paired import (
+    ComparisonArm,
+    ComparisonPolicy,
+    CompatibilityReport,
+    PairedComparisonEngine,
+    PairedComparisonResult,
+)
+from openbench.performance import (
+    PerformanceMonitor,
+    PerformanceReport,
+    PricingConfig,
+    RequestTelemetry,
+    compute_percentile,
+)
+from openbench.preferences import (
+    AnnotationImporter,
+    PreferenceAnalyzer,
+    PreferenceAnnotation,
+    PreferenceTaskBuilder,
+)
+from openbench.quality import AuditFinding, AuditReport, DatasetAuditor, ExperimentAuditor
+from openbench.reporting import ReportGenerator, escape_markdown_cell
+from openbench.security import (
+    redact_prompt_if_private,
+    redact_sensitive_data,
+    resolve_env_credential,
+    sanitize_error_message,
+    validate_endpoint_url,
+    validate_safe_output_path,
+)
+
+__version__ = "0.4.0"
+
+__all__ = [
+    "__version__",
+    "BaseModelAdapter",
+    "BenchmarkAnalytics",
+    "CacheRecord",
+    "ComparisonResult",
+    "ConfigurationError",
+    "Dataset",
+    "DatasetCatalog",
+    "DatasetCatalogEntry",
+    "DatasetLoader",
+    "DatasetRecord",
+    "DatasetValidationError",
+    "DeterministicLookupAdapter",
+    "EmptyDatasetError",
+    "EvaluationConfig",
+    "EvaluationProfile",
+    "EvaluationResult",
+    "Evaluator",
+    "ExecutionConfig",
+    "ExecutionEngine",
+    "ExecutionOutcome",
+    "ExperimentManager",
+    "ExperimentRecord",
+    "GenerationConfig",
+    "HistoryEntry",
+    "HuggingFaceAdapter",
+    "IdFilterSpec",
+    "IntegrityCheckResult",
+    "ItemEvaluationResult",
+    "LexicalSimilarityMetric",
+    "LocalCallableAdapter",
+    "MetricError",
+    "MetricLeader",
+    "MockModelProvider",
+    "ModelComparator",
+    "ModelComparisonEntry",
+    "ModelResponseSet",
+    "OllamaAdapter",
+    "OllamaConnectionError",
+    "OpenBenchError",
+    "PROVIDER_REGISTRY",
+    "PerformanceMonitor",
+    "PerformanceReport",
+    "PricingConfig",
+    "ProfileLoader",
+    "ProviderHealthStatus",
+    "ReportGenerator",
+    "RequestTelemetry",
+    "ResponseCache",
+    "ResponseItem",
+    "ResponseLoader",
+    "ResponseValidationError",
+    "ResultHistoryStore",
+    "RetryPolicy",
+    "RunIntegrityVerifier",
+    "classify_execution_error",
+    "compute_dict_sha256",
+    "compute_file_sha256",
+    "compute_percentile",
+    "create_provider",
+    "escape_markdown_cell",
+    "list_registered_providers",
+    "load_ids_file",
+    "redact_prompt_if_private",
+    "redact_sensitive_data",
+    "register_provider",
+    "resolve_env_credential",
+    "sanitize_error_message",
+    "validate_endpoint_url",
+    "validate_safe_output_path",
+    # v0.4.0 additions (the 70 v0.3.1 names above are unchanged and keep their order)
+    "ComparisonArm",
+    "ComparisonPolicy",
+    "CompatibilityReport",
+    "IncompatibleComparisonError",
+    "PairedComparisonEngine",
+    "PairedComparisonResult",
+    "RepeatGroupResult",
+    "RunLineage",
+    "AnnotationImporter",
+    "PreferenceAnalyzer",
+    "PreferenceAnnotation",
+    "PreferenceTaskBuilder",
+    "AuditFinding",
+    "AuditReport",
+    "DatasetAuditor",
+    "ExperimentAuditor",
+]

openbench/adapters/__init__.py

@@ -0,0 +1,39 @@
+"""Model providers and adapters package for OpenBench AI."""
+
+from openbench.adapters.base import (
+    PROVIDER_REGISTRY,
+    BaseModelAdapter,
+    GenerationConfig,
+    ProviderHealthStatus,
+    create_provider,
+    list_registered_providers,
+    register_provider,
+)
+from openbench.adapters.huggingface import HuggingFaceAdapter
+from openbench.adapters.local import DeterministicLookupAdapter, LocalCallableAdapter
+from openbench.adapters.mock import MockModelProvider
+from openbench.adapters.ollama import DEFAULT_OLLAMA_ENDPOINT, OllamaAdapter, OllamaConnectionError
+
+# Register all built-in providers
+register_provider(MockModelProvider)
+register_provider(OllamaAdapter)
+register_provider(HuggingFaceAdapter, aliases=["huggingface", "hf"])
+register_provider(LocalCallableAdapter)
+register_provider(DeterministicLookupAdapter)
+
+__all__ = [
+    "BaseModelAdapter",
+    "DEFAULT_OLLAMA_ENDPOINT",
+    "DeterministicLookupAdapter",
+    "GenerationConfig",
+    "HuggingFaceAdapter",
+    "LocalCallableAdapter",
+    "MockModelProvider",
+    "OllamaAdapter",
+    "OllamaConnectionError",
+    "PROVIDER_REGISTRY",
+    "ProviderHealthStatus",
+    "create_provider",
+    "list_registered_providers",
+    "register_provider",
+]

openbench/adapters/base.py

@@ -0,0 +1,220 @@
+"""Base model inference adapter interface, generation config, and provider registry for OpenBench AI."""
+
+from __future__ import annotations
+
+import inspect
+from abc import ABC, abstractmethod
+from dataclasses import dataclass, field
+from typing import Any
+
+from openbench.dataset import Dataset, DatasetRecord
+from openbench.evaluator import ModelResponseSet, ResponseItem
+from openbench.exceptions import ConfigurationError
+from openbench.security import redact_sensitive_data
+
+
+@dataclass(frozen=True)
+class GenerationConfig:
+    """Standardized generation parameters passed to model providers."""
+
+    temperature: float = 0.0
+    max_tokens: int = 256
+    top_p: float = 1.0
+    seed: int | None = None
+    stop_sequences: list[str] = field(default_factory=list)
+    stream: bool = False
+    extra_options: dict[str, Any] = field(default_factory=dict)
+
+    def to_dict(self) -> dict[str, Any]:
+        payload: dict[str, Any] = {
+            "temperature": self.temperature,
+            "max_tokens": self.max_tokens,
+            "top_p": self.top_p,
+            "stream": self.stream,
+        }
+        if self.seed is not None:
+            payload["seed"] = self.seed
+        if self.stop_sequences:
+            payload["stop_sequences"] = list(self.stop_sequences)
+        if self.extra_options:
+            payload["extra_options"] = redact_sensitive_data(dict(self.extra_options))
+        return payload
+
+
+@dataclass(frozen=True)
+class ProviderHealthStatus:
+    """Result of checking a model provider's availability and configuration."""
+
+    provider_name: str
+    available: bool
+    message: str
+    details: dict[str, Any] = field(default_factory=dict)
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "provider_name": self.provider_name,
+            "available": self.available,
+            "message": self.message,
+            "details": redact_sensitive_data(dict(self.details)),
+        }
+
+
+class BaseModelAdapter(ABC):
+    """Abstract interface for connecting local or external language models to OpenBench AI.
+
+    Design Principles:
+    - Adapters are strictly optional; core OpenBench evaluation works directly on JSON files.
+    - Never hardcode tokens or automatically call paid APIs.
+    - Record reproducibility metadata (`model_name`, `model_version`, `model_provider`,
+      `inference_config`) alongside generated responses.
+    """
+
+    provider_name: str = "base_adapter"
+    description: str = "Abstract base model provider adapter."
+    is_experimental: bool = False
+    is_mock: bool = False
+    optional_dependencies: tuple[str, ...] = ()
+    #: True only for adapters that place ``GenerationConfig.seed`` in the provider request
+    #: (v0.4.0). A provider may still ignore the seed or be non-deterministic.
+    supports_generation_seed: bool = False
+
+    def __init__(
+        self,
+        model_name: str,
+        model_version: str = "1.0",
+        inference_config: dict[str, Any] | None = None,
+        generation_config: GenerationConfig | None = None,
+        timeout_seconds: float = 30.0,
+    ) -> None:
+        if not model_name or not model_name.strip():
+            raise ValueError("model_name must be a non-empty string")
+        if timeout_seconds <= 0:
+            raise ConfigurationError("timeout_seconds must be positive")
+        self.model_name = model_name.strip()
+        self.model_version = model_version.strip() or "1.0"
+        self.generation_config = generation_config or GenerationConfig()
+        self.timeout_seconds = float(timeout_seconds)
+        base_inf = self.generation_config.to_dict()
+        if inference_config:
+            base_inf.update(redact_sensitive_data(dict(inference_config)))
+        self.inference_config = base_inf
+        #: Number of provider requests into which this adapter placed a generation seed.
+        self.seed_forward_count = 0
+
+    def with_generation_seed(self, seed: int | None) -> BaseModelAdapter:
+        """Return a shallow copy of this adapter whose generation seed is ``seed`` (v0.4.0).
+
+        Used for per-repetition seeds. The copy shares any client or transport objects with
+        the original; only ``generation_config``, ``inference_config`` and the seed counter
+        are replaced.
+        """
+        import copy
+        from dataclasses import replace
+
+        clone = copy.copy(self)
+        clone.generation_config = replace(self.generation_config, seed=seed)
+        inference = dict(self.inference_config)
+        if seed is None:
+            inference.pop("seed", None)
+        else:
+            inference["seed"] = seed
+        clone.inference_config = inference
+        clone.seed_forward_count = 0
+        return clone
+
+    @abstractmethod
+    def generate_one(self, record: DatasetRecord) -> ResponseItem:
+        """Generate a single ResponseItem for a given DatasetRecord."""
+
+    def generate_responses(self, dataset: Dataset) -> ModelResponseSet:
+        """Run inference across all records in a Dataset and return a ModelResponseSet."""
+        items = [self.generate_one(record) for record in dataset.records]
+        return ModelResponseSet(
+            model_name=self.model_name,
+            model_version=self.model_version,
+            responses=items,
+            model_provider=self.provider_name,
+            inference_config=self.reproducibility_metadata(),
+        )
+
+    def check_connection(self) -> ProviderHealthStatus:
+        """Verify whether the provider is reachable and properly configured."""
+        return ProviderHealthStatus(
+            provider_name=self.provider_name,
+            available=True,
+            message=f"Provider '{self.provider_name}' is ready.",
+            details={"model_name": self.model_name, "is_mock": self.is_mock},
+        )
+
+    def reproducibility_metadata(self) -> dict[str, Any]:
+        """Return safe reproducibility metadata for reports and cards (with secrets redacted)."""
+        return redact_sensitive_data(
+            {
+                "provider": self.provider_name,
+                "model_name": self.model_name,
+                "model_version": self.model_version,
+                "experimental": self.is_experimental,
+                "is_mock": self.is_mock,
+                "timeout_seconds": self.timeout_seconds,
+                **self.inference_config,
+            }
+        )
+
+
+PROVIDER_REGISTRY: dict[str, type[BaseModelAdapter]] = {}
+
+
+def register_provider(adapter_cls: type[BaseModelAdapter], aliases: list[str] | None = None) -> None:
+    """Register a BaseModelAdapter subclass in `PROVIDER_REGISTRY`."""
+    if not isinstance(adapter_cls, type) or not issubclass(adapter_cls, BaseModelAdapter):
+        raise ConfigurationError("Provider class must inherit from BaseModelAdapter")
+    p_name = getattr(adapter_cls, "provider_name", "").strip().lower()
+    if not p_name:
+        raise ConfigurationError("Provider class must define a non-empty 'provider_name'")
+    PROVIDER_REGISTRY[p_name] = adapter_cls
+    for alias in aliases or []:
+        PROVIDER_REGISTRY[alias.strip().lower()] = adapter_cls
+
+
+def create_provider(provider_name: str, **kwargs: Any) -> BaseModelAdapter:
+    """Instantiate a registered provider adapter by name."""
+    key = provider_name.strip().lower()
+    if key not in PROVIDER_REGISTRY:
+        available = sorted({cls.provider_name for cls in PROVIDER_REGISTRY.values()})
+        raise ConfigurationError(
+            f"Unknown model provider '{provider_name}'. Registered providers: {', '.join(available)}"
+        )
+    adapter_cls = PROVIDER_REGISTRY[key]
+    sig = inspect.signature(adapter_cls.__init__)
+    accepts_var_kw = any(
+        p.kind == inspect.Parameter.VAR_KEYWORD for p in sig.parameters.values()
+    )
+    filtered_kwargs: dict[str, Any] = {}
+    for k, v in kwargs.items():
+        if accepts_var_kw or k in sig.parameters:
+            filtered_kwargs[k] = v
+        else:
+            raise ConfigurationError(
+                f"Provider '{provider_name}' does not accept configuration argument '{k}'"
+            )
+    return adapter_cls(**filtered_kwargs)
+
+
+def list_registered_providers() -> list[dict[str, Any]]:
+    """Return metadata dictionaries for all unique registered model providers."""
+    unique: dict[str, type[BaseModelAdapter]] = {}
+    for cls in PROVIDER_REGISTRY.values():
+        unique[cls.provider_name] = cls
+    result: list[dict[str, Any]] = []
+    for name in sorted(unique):
+        cls = unique[name]
+        result.append(
+            {
+                "provider_name": cls.provider_name,
+                "description": cls.description,
+                "is_experimental": cls.is_experimental,
+                "is_mock": cls.is_mock,
+                "optional_dependencies": list(cls.optional_dependencies),
+            }
+        )
+    return result

openbench/adapters/huggingface.py

@@ -0,0 +1,158 @@
+"""Optional Hugging Face `transformers` adapter for OpenBench AI."""
+
+from __future__ import annotations
+
+import time
+from typing import Any
+
+from openbench.adapters.base import BaseModelAdapter, GenerationConfig, ProviderHealthStatus
+from openbench.dataset import DatasetRecord
+from openbench.evaluator import ResponseItem
+from openbench.exceptions import ConfigurationError, OpenBenchError
+
+
+class HuggingFaceAdapter(BaseModelAdapter):
+    """Optional adapter for local Hugging Face `transformers` text-generation pipelines.
+
+    Status: Experimental (requires optional `transformers` and `torch` packages).
+
+    Safety & Reproducibility Guarantees:
+    - Does NOT import `transformers` at module import time.
+    - Defaults to `local_files_only=True` so weights are NEVER automatically downloaded
+      from the Hugging Face Hub unless the caller explicitly sets `allow_download=True`.
+    - Contains zero hardcoded API tokens and makes zero paid API calls.
+    - Records explicit `model_id`, `revision`, `device`, and generation parameters.
+    - Catches `MemoryError` and CUDA/CPU OOM errors with actionable remediation hints.
+    """
+
+    provider_name = "huggingface_transformers"
+    description = "Local Hugging Face transformers text-generation pipeline adapter (Experimental)."
+    is_experimental = True
+    is_mock = False
+    optional_dependencies = ("transformers", "torch")
+
+    def __init__(
+        self,
+        model_id: str = "gpt2",
+        revision: str = "main",
+        allow_download: bool = False,
+        device: str = "cpu",
+        max_new_tokens: int = 128,
+        temperature: float = 0.0,
+        pipeline_fn: Any | None = None,
+        generation_config: GenerationConfig | None = None,
+        timeout_seconds: float = 60.0,
+        model_name: str | None = None,
+    ) -> None:
+        resolved_id = (model_name or model_id).strip()
+        if not resolved_id:
+            raise ConfigurationError("An explicit Hugging Face model_id is required")
+        gen_cfg = generation_config or GenerationConfig(
+            temperature=temperature,
+            max_tokens=max_new_tokens,
+        )
+        super().__init__(
+            model_name=resolved_id,
+            model_version=revision.strip() or "main",
+            generation_config=gen_cfg,
+            timeout_seconds=timeout_seconds,
+            inference_config={
+                "model_id": resolved_id,
+                "revision": revision.strip() or "main",
+                "local_files_only": not allow_download,
+                "device": device,
+                "max_new_tokens": max_new_tokens,
+                "temperature": temperature,
+            },
+        )
+        self.model_id = resolved_id
+        self.revision = revision.strip() or "main"
+        self.allow_download = allow_download
+        self.device = device
+        self.max_new_tokens = max_new_tokens
+        self.temperature = temperature
+        self._pipeline = pipeline_fn
+
+    def check_connection(self) -> ProviderHealthStatus:
+        if self._pipeline is not None:
+            return ProviderHealthStatus(
+                provider_name=self.provider_name,
+                available=True,
+                message=f"HuggingFaceAdapter ready with injected pipeline for '{self.model_id}'.",
+                details={"model_id": self.model_id, "revision": self.revision},
+            )
+        try:
+            import transformers  # type: ignore[import-not-found] # noqa: F401
+        except ImportError:
+            return ProviderHealthStatus(
+                provider_name=self.provider_name,
+                available=False,
+                message=(
+                    "Optional dependency 'transformers' is not installed. "
+                    "Install it with `pip install -e '.[huggingface]'`."
+                ),
+                details={"model_id": self.model_id, "device": self.device},
+            )
+        return ProviderHealthStatus(
+            provider_name=self.provider_name,
+            available=True,
+            message=f"transformers package is installed (model='{self.model_id}', local_files_only={not self.allow_download}).",
+            details={"model_id": self.model_id, "revision": self.revision, "device": self.device},
+        )
+
+    def _get_pipeline(self) -> Any:
+        if self._pipeline is not None:
+            return self._pipeline
+        try:
+            from transformers import pipeline  # type: ignore[import-not-found]
+        except ImportError as exc:
+            raise ConfigurationError(
+                "Optional dependency 'transformers' is not installed. "
+                "Install it with `pip install -e '.[huggingface]'` to use HuggingFaceAdapter."
+            ) from exc
+
+        try:
+            self._pipeline = pipeline(
+                "text-generation",
+                model=self.model_id,
+                revision=self.revision,
+                device=self.device,
+                model_kwargs={"local_files_only": not self.allow_download},
+            )
+        except MemoryError as exc:
+            raise OpenBenchError(
+                f"Insufficient system memory to load Hugging Face model '{self.model_id}' on device '{self.device}'."
+            ) from exc
+        except Exception as exc:
+            raise OpenBenchError(
+                f"Failed to load Hugging Face model '{self.model_id}' (revision='{self.revision}', "
+                f"local_files_only={not self.allow_download}): {exc}"
+            ) from exc
+        return self._pipeline
+
+    def generate_one(self, record: DatasetRecord) -> ResponseItem:
+        pipe = self._get_pipeline()
+        start = time.perf_counter()
+        try:
+            outputs = pipe(
+                record.prompt,
+                max_new_tokens=self.max_new_tokens,
+                do_sample=self.temperature > 0.0,
+                return_full_text=False,
+            )
+        except MemoryError as exc:
+            raise OpenBenchError(
+                f"Out of memory while generating response for '{record.id}' with model '{self.model_id}'."
+            ) from exc
+        elapsed_ms = (time.perf_counter() - start) * 1000.0
+
+        if isinstance(outputs, list) and outputs and isinstance(outputs[0], dict):
+            generated_text = str(outputs[0].get("generated_text", ""))
+        else:
+            generated_text = str(outputs)
+
+        return ResponseItem(
+            id=record.id,
+            response=generated_text,
+            latency_ms=round(elapsed_ms, 3),
+        )