compactbench 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- compactbench/__init__.py +10 -0
- compactbench/__main__.py +6 -0
- compactbench/cli.py +343 -0
- compactbench/compactors/__init__.py +47 -0
- compactbench/compactors/_state_parser.py +118 -0
- compactbench/compactors/_utils.py +37 -0
- compactbench/compactors/base.py +38 -0
- compactbench/compactors/errors.py +11 -0
- compactbench/compactors/hierarchical_summary.py +113 -0
- compactbench/compactors/hybrid_ledger.py +115 -0
- compactbench/compactors/naive_summary.py +54 -0
- compactbench/compactors/structured_state.py +64 -0
- compactbench/config.py +36 -0
- compactbench/contracts/__init__.py +41 -0
- compactbench/contracts/artifact.py +49 -0
- compactbench/contracts/case.py +81 -0
- compactbench/contracts/result.py +79 -0
- compactbench/dsl/__init__.py +66 -0
- compactbench/dsl/errors.py +23 -0
- compactbench/dsl/generators.py +243 -0
- compactbench/dsl/models.py +107 -0
- compactbench/dsl/parser.py +55 -0
- compactbench/dsl/substitution.py +32 -0
- compactbench/dsl/validator.py +97 -0
- compactbench/engine/__init__.py +29 -0
- compactbench/engine/difficulty.py +54 -0
- compactbench/engine/distractors.py +69 -0
- compactbench/engine/errors.py +11 -0
- compactbench/engine/generation.py +96 -0
- compactbench/engine/seeds.py +31 -0
- compactbench/engine/transcript.py +84 -0
- compactbench/leaderboard/__init__.py +38 -0
- compactbench/leaderboard/errors.py +11 -0
- compactbench/leaderboard/projection.py +103 -0
- compactbench/leaderboard/qualification.py +90 -0
- compactbench/leaderboard/ranking.py +71 -0
- compactbench/providers/__init__.py +52 -0
- compactbench/providers/_retry.py +48 -0
- compactbench/providers/base.py +44 -0
- compactbench/providers/errors.py +15 -0
- compactbench/providers/google_ai_studio.py +112 -0
- compactbench/providers/groq.py +95 -0
- compactbench/providers/mock.py +53 -0
- compactbench/providers/ollama.py +100 -0
- compactbench/py.typed +0 -0
- compactbench/runner/__init__.py +53 -0
- compactbench/runner/_resolver.py +57 -0
- compactbench/runner/continuation.py +78 -0
- compactbench/runner/cycle.py +67 -0
- compactbench/runner/errors.py +15 -0
- compactbench/runner/evaluation.py +71 -0
- compactbench/runner/persistence.py +227 -0
- compactbench/runner/run.py +257 -0
- compactbench/scoring/__init__.py +48 -0
- compactbench/scoring/checks.py +92 -0
- compactbench/scoring/compression.py +66 -0
- compactbench/scoring/contradictions.py +70 -0
- compactbench/scoring/drift.py +30 -0
- compactbench/scoring/errors.py +7 -0
- compactbench/scoring/scorer.py +77 -0
- compactbench/utils/__init__.py +1 -0
- compactbench-0.1.0.dist-info/METADATA +161 -0
- compactbench-0.1.0.dist-info/RECORD +66 -0
- compactbench-0.1.0.dist-info/WHEEL +4 -0
- compactbench-0.1.0.dist-info/entry_points.txt +2 -0
- compactbench-0.1.0.dist-info/licenses/LICENSE +201 -0
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
"""Model provider abstractions and concrete clients."""
|
|
2
|
+
|
|
3
|
+
from compactbench.providers.base import (
|
|
4
|
+
CompletionRequest,
|
|
5
|
+
CompletionResponse,
|
|
6
|
+
Provider,
|
|
7
|
+
)
|
|
8
|
+
from compactbench.providers.errors import (
|
|
9
|
+
ProviderError,
|
|
10
|
+
ProviderResponseError,
|
|
11
|
+
UnknownProviderError,
|
|
12
|
+
)
|
|
13
|
+
from compactbench.providers.google_ai_studio import GoogleAIStudioProvider
|
|
14
|
+
from compactbench.providers.groq import GroqProvider
|
|
15
|
+
from compactbench.providers.mock import MockProvider
|
|
16
|
+
from compactbench.providers.ollama import OllamaProvider
|
|
17
|
+
|
|
18
|
+
_REGISTRY: dict[str, type[Provider]] = {
|
|
19
|
+
MockProvider.key: MockProvider,
|
|
20
|
+
GroqProvider.key: GroqProvider,
|
|
21
|
+
GoogleAIStudioProvider.key: GoogleAIStudioProvider,
|
|
22
|
+
OllamaProvider.key: OllamaProvider,
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def list_providers() -> list[str]:
|
|
27
|
+
"""Return the sorted list of registered provider keys."""
|
|
28
|
+
return sorted(_REGISTRY)
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def get_provider_cls(key: str) -> type[Provider]:
|
|
32
|
+
"""Return the provider class for ``key`` or raise :class:`UnknownProviderError`."""
|
|
33
|
+
cls = _REGISTRY.get(key)
|
|
34
|
+
if cls is None:
|
|
35
|
+
raise UnknownProviderError(f"unknown provider {key!r}. Known: {sorted(_REGISTRY)}")
|
|
36
|
+
return cls
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
__all__ = [
|
|
40
|
+
"CompletionRequest",
|
|
41
|
+
"CompletionResponse",
|
|
42
|
+
"GoogleAIStudioProvider",
|
|
43
|
+
"GroqProvider",
|
|
44
|
+
"MockProvider",
|
|
45
|
+
"OllamaProvider",
|
|
46
|
+
"Provider",
|
|
47
|
+
"ProviderError",
|
|
48
|
+
"ProviderResponseError",
|
|
49
|
+
"UnknownProviderError",
|
|
50
|
+
"get_provider_cls",
|
|
51
|
+
"list_providers",
|
|
52
|
+
]
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
"""Shared async exponential-backoff retry helper for providers.
|
|
2
|
+
|
|
3
|
+
Each provider decides which of its SDK's exceptions are retryable (typically
|
|
4
|
+
rate-limit and transient network errors) and hands a predicate plus the async
|
|
5
|
+
call to :func:`retry_with_backoff`.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import asyncio
|
|
11
|
+
from collections.abc import Awaitable, Callable
|
|
12
|
+
from typing import TypeVar
|
|
13
|
+
|
|
14
|
+
T = TypeVar("T")
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
async def retry_with_backoff(
|
|
18
|
+
operation: Callable[[], Awaitable[T]],
|
|
19
|
+
*,
|
|
20
|
+
is_retryable: Callable[[Exception], bool],
|
|
21
|
+
max_retries: int = 3,
|
|
22
|
+
base_delay: float = 2.0,
|
|
23
|
+
max_delay: float = 30.0,
|
|
24
|
+
) -> T:
|
|
25
|
+
"""Call ``operation`` with exponential backoff on retryable exceptions.
|
|
26
|
+
|
|
27
|
+
Returns the operation's result on first success. On a retryable failure,
|
|
28
|
+
waits ``min(base_delay * 2**attempt, max_delay)`` seconds and retries.
|
|
29
|
+
After ``max_retries`` failed attempts the last exception is re-raised.
|
|
30
|
+
Non-retryable exceptions are re-raised immediately.
|
|
31
|
+
"""
|
|
32
|
+
if max_retries < 1:
|
|
33
|
+
raise ValueError(f"max_retries must be >= 1, got {max_retries}")
|
|
34
|
+
|
|
35
|
+
last_exc: Exception | None = None
|
|
36
|
+
for attempt in range(max_retries):
|
|
37
|
+
try:
|
|
38
|
+
return await operation()
|
|
39
|
+
except Exception as exc:
|
|
40
|
+
if not is_retryable(exc):
|
|
41
|
+
raise
|
|
42
|
+
last_exc = exc
|
|
43
|
+
if attempt < max_retries - 1:
|
|
44
|
+
delay = min(base_delay * (2**attempt), max_delay)
|
|
45
|
+
await asyncio.sleep(delay)
|
|
46
|
+
|
|
47
|
+
assert last_exc is not None # unreachable unless last_exc was set in the loop
|
|
48
|
+
raise last_exc
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
"""Model provider ABC and request/response shapes.
|
|
2
|
+
|
|
3
|
+
Concrete implementations (Groq, Google AI Studio, Ollama) live in WO-006.
|
|
4
|
+
The mock provider below is part of WO-005 so compactors can be tested offline.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
from abc import ABC, abstractmethod
|
|
10
|
+
from dataclasses import dataclass, field
|
|
11
|
+
from typing import Any, ClassVar
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
@dataclass(frozen=True)
|
|
15
|
+
class CompletionRequest:
|
|
16
|
+
"""A normalized request passed to a provider's ``complete`` method."""
|
|
17
|
+
|
|
18
|
+
model: str
|
|
19
|
+
prompt: str
|
|
20
|
+
system: str | None = None
|
|
21
|
+
max_tokens: int = 2048
|
|
22
|
+
temperature: float = 0.0
|
|
23
|
+
response_format: dict[str, Any] | None = None
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
@dataclass(frozen=True)
|
|
27
|
+
class CompletionResponse:
|
|
28
|
+
"""A normalized response returned from a provider."""
|
|
29
|
+
|
|
30
|
+
text: str
|
|
31
|
+
prompt_tokens: int
|
|
32
|
+
completion_tokens: int
|
|
33
|
+
model: str
|
|
34
|
+
raw: dict[str, Any] = field(default_factory=dict[str, Any])
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
class Provider(ABC):
|
|
38
|
+
"""Base class every provider must implement."""
|
|
39
|
+
|
|
40
|
+
key: ClassVar[str]
|
|
41
|
+
|
|
42
|
+
@abstractmethod
|
|
43
|
+
async def complete(self, request: CompletionRequest) -> CompletionResponse:
|
|
44
|
+
"""Issue a completion request and return a normalized response."""
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
"""Provider-specific exceptions."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
class ProviderError(Exception):
|
|
7
|
+
"""Base exception for provider operations."""
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class UnknownProviderError(ProviderError):
|
|
11
|
+
"""Requested provider key is not registered."""
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class ProviderResponseError(ProviderError):
|
|
15
|
+
"""Provider returned a malformed or unusable response."""
|
|
@@ -0,0 +1,112 @@
|
|
|
1
|
+
"""Google AI Studio provider (Gemini 2.0 Flash, etc.).
|
|
2
|
+
|
|
3
|
+
Requires ``compactbench[providers]`` (installs the ``google-genai`` SDK) and a
|
|
4
|
+
``COMPACTBENCH_GOOGLE_AI_STUDIO_API_KEY`` environment variable.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import os
|
|
10
|
+
from typing import Any, ClassVar, cast
|
|
11
|
+
|
|
12
|
+
from compactbench.providers._retry import retry_with_backoff
|
|
13
|
+
from compactbench.providers.base import (
|
|
14
|
+
CompletionRequest,
|
|
15
|
+
CompletionResponse,
|
|
16
|
+
Provider,
|
|
17
|
+
)
|
|
18
|
+
from compactbench.providers.errors import ProviderError
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
class GoogleAIStudioProvider(Provider):
|
|
22
|
+
"""Async Google AI Studio client (via the google-genai SDK)."""
|
|
23
|
+
|
|
24
|
+
key: ClassVar[str] = "google-ai-studio"
|
|
25
|
+
|
|
26
|
+
def __init__(
|
|
27
|
+
self,
|
|
28
|
+
*,
|
|
29
|
+
api_key: str | None = None,
|
|
30
|
+
max_retries: int = 3,
|
|
31
|
+
base_backoff_seconds: float = 2.0,
|
|
32
|
+
) -> None:
|
|
33
|
+
try:
|
|
34
|
+
from google import genai
|
|
35
|
+
except ImportError as exc:
|
|
36
|
+
raise ProviderError(
|
|
37
|
+
"google-genai SDK is not installed. "
|
|
38
|
+
"Install with: pip install 'compactbench[providers]'"
|
|
39
|
+
) from exc
|
|
40
|
+
|
|
41
|
+
resolved_key = api_key or os.environ.get("COMPACTBENCH_GOOGLE_AI_STUDIO_API_KEY")
|
|
42
|
+
if not resolved_key:
|
|
43
|
+
raise ProviderError(
|
|
44
|
+
"Google AI Studio API key required. Set "
|
|
45
|
+
"COMPACTBENCH_GOOGLE_AI_STUDIO_API_KEY or pass api_key=."
|
|
46
|
+
)
|
|
47
|
+
|
|
48
|
+
self._client: Any = genai.Client(api_key=resolved_key)
|
|
49
|
+
self._max_retries = max_retries
|
|
50
|
+
self._base_backoff_seconds = base_backoff_seconds
|
|
51
|
+
|
|
52
|
+
async def complete(self, request: CompletionRequest) -> CompletionResponse:
|
|
53
|
+
from google.genai import errors as genai_errors
|
|
54
|
+
from google.genai import types
|
|
55
|
+
|
|
56
|
+
def _is_retryable(exc: Exception) -> bool:
|
|
57
|
+
if not isinstance(exc, genai_errors.APIError):
|
|
58
|
+
return False
|
|
59
|
+
code = getattr(exc, "code", None)
|
|
60
|
+
if code is None:
|
|
61
|
+
code = getattr(exc, "status_code", None)
|
|
62
|
+
if not isinstance(code, int):
|
|
63
|
+
return False
|
|
64
|
+
return code == 429 or code >= 500
|
|
65
|
+
|
|
66
|
+
config_kwargs: dict[str, Any] = {
|
|
67
|
+
"temperature": request.temperature,
|
|
68
|
+
"max_output_tokens": request.max_tokens,
|
|
69
|
+
}
|
|
70
|
+
if request.system:
|
|
71
|
+
config_kwargs["system_instruction"] = request.system
|
|
72
|
+
if request.response_format and request.response_format.get("type") == "json_object":
|
|
73
|
+
config_kwargs["response_mime_type"] = "application/json"
|
|
74
|
+
config = types.GenerateContentConfig(**config_kwargs)
|
|
75
|
+
|
|
76
|
+
async def _call() -> Any:
|
|
77
|
+
return await self._client.aio.models.generate_content(
|
|
78
|
+
model=request.model,
|
|
79
|
+
contents=request.prompt,
|
|
80
|
+
config=config,
|
|
81
|
+
)
|
|
82
|
+
|
|
83
|
+
response = await retry_with_backoff(
|
|
84
|
+
_call,
|
|
85
|
+
is_retryable=_is_retryable,
|
|
86
|
+
max_retries=self._max_retries,
|
|
87
|
+
base_delay=self._base_backoff_seconds,
|
|
88
|
+
)
|
|
89
|
+
|
|
90
|
+
text = getattr(response, "text", None) or ""
|
|
91
|
+
usage_metadata = getattr(response, "usage_metadata", None)
|
|
92
|
+
prompt_tokens = (
|
|
93
|
+
(getattr(usage_metadata, "prompt_token_count", 0) or 0) if usage_metadata else 0
|
|
94
|
+
)
|
|
95
|
+
completion_tokens = (
|
|
96
|
+
(getattr(usage_metadata, "candidates_token_count", 0) or 0) if usage_metadata else 0
|
|
97
|
+
)
|
|
98
|
+
raw_candidates = getattr(response, "candidates", None)
|
|
99
|
+
candidates: list[Any] = (
|
|
100
|
+
cast("list[Any]", raw_candidates) if isinstance(raw_candidates, list) else []
|
|
101
|
+
)
|
|
102
|
+
finish_reason = getattr(candidates[0], "finish_reason", None) if candidates else None
|
|
103
|
+
return CompletionResponse(
|
|
104
|
+
text=text,
|
|
105
|
+
prompt_tokens=prompt_tokens,
|
|
106
|
+
completion_tokens=completion_tokens,
|
|
107
|
+
model=request.model,
|
|
108
|
+
raw={
|
|
109
|
+
"provider": "google-ai-studio",
|
|
110
|
+
"finish_reason": str(finish_reason) if finish_reason is not None else None,
|
|
111
|
+
},
|
|
112
|
+
)
|
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
"""Groq Cloud provider (Llama 3.3 70B, Kimi K2, etc.).
|
|
2
|
+
|
|
3
|
+
Requires ``compactbench[providers]`` (installs the ``groq`` SDK) and a
|
|
4
|
+
``COMPACTBENCH_GROQ_API_KEY`` environment variable.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import os
|
|
10
|
+
from typing import Any, ClassVar
|
|
11
|
+
|
|
12
|
+
from compactbench.providers._retry import retry_with_backoff
|
|
13
|
+
from compactbench.providers.base import (
|
|
14
|
+
CompletionRequest,
|
|
15
|
+
CompletionResponse,
|
|
16
|
+
Provider,
|
|
17
|
+
)
|
|
18
|
+
from compactbench.providers.errors import ProviderError, ProviderResponseError
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
class GroqProvider(Provider):
|
|
22
|
+
"""Async Groq client with rate-limit backoff."""
|
|
23
|
+
|
|
24
|
+
key: ClassVar[str] = "groq"
|
|
25
|
+
|
|
26
|
+
def __init__(
|
|
27
|
+
self,
|
|
28
|
+
*,
|
|
29
|
+
api_key: str | None = None,
|
|
30
|
+
max_retries: int = 3,
|
|
31
|
+
base_backoff_seconds: float = 2.0,
|
|
32
|
+
) -> None:
|
|
33
|
+
try:
|
|
34
|
+
from groq import AsyncGroq
|
|
35
|
+
except ImportError as exc:
|
|
36
|
+
raise ProviderError(
|
|
37
|
+
"groq SDK is not installed. Install with: pip install 'compactbench[providers]'"
|
|
38
|
+
) from exc
|
|
39
|
+
|
|
40
|
+
resolved_key = api_key or os.environ.get("COMPACTBENCH_GROQ_API_KEY")
|
|
41
|
+
if not resolved_key:
|
|
42
|
+
raise ProviderError(
|
|
43
|
+
"Groq API key required. Set COMPACTBENCH_GROQ_API_KEY or pass api_key=."
|
|
44
|
+
)
|
|
45
|
+
|
|
46
|
+
self._client: Any = AsyncGroq(api_key=resolved_key)
|
|
47
|
+
self._max_retries = max_retries
|
|
48
|
+
self._base_backoff_seconds = base_backoff_seconds
|
|
49
|
+
|
|
50
|
+
async def complete(self, request: CompletionRequest) -> CompletionResponse:
|
|
51
|
+
from groq import APIConnectionError, APITimeoutError, RateLimitError
|
|
52
|
+
|
|
53
|
+
def _is_retryable(exc: Exception) -> bool:
|
|
54
|
+
return isinstance(exc, RateLimitError | APITimeoutError | APIConnectionError)
|
|
55
|
+
|
|
56
|
+
messages: list[dict[str, str]] = []
|
|
57
|
+
if request.system:
|
|
58
|
+
messages.append({"role": "system", "content": request.system})
|
|
59
|
+
messages.append({"role": "user", "content": request.prompt})
|
|
60
|
+
|
|
61
|
+
kwargs: dict[str, Any] = {
|
|
62
|
+
"model": request.model,
|
|
63
|
+
"messages": messages,
|
|
64
|
+
"temperature": request.temperature,
|
|
65
|
+
"max_tokens": request.max_tokens,
|
|
66
|
+
}
|
|
67
|
+
if request.response_format:
|
|
68
|
+
kwargs["response_format"] = request.response_format
|
|
69
|
+
|
|
70
|
+
async def _call() -> Any:
|
|
71
|
+
return await self._client.chat.completions.create(**kwargs)
|
|
72
|
+
|
|
73
|
+
response = await retry_with_backoff(
|
|
74
|
+
_call,
|
|
75
|
+
is_retryable=_is_retryable,
|
|
76
|
+
max_retries=self._max_retries,
|
|
77
|
+
base_delay=self._base_backoff_seconds,
|
|
78
|
+
)
|
|
79
|
+
|
|
80
|
+
if not response.choices:
|
|
81
|
+
raise ProviderResponseError("Groq returned no choices in response")
|
|
82
|
+
choice = response.choices[0]
|
|
83
|
+
text = choice.message.content or ""
|
|
84
|
+
usage = response.usage
|
|
85
|
+
return CompletionResponse(
|
|
86
|
+
text=text,
|
|
87
|
+
prompt_tokens=usage.prompt_tokens if usage else 0,
|
|
88
|
+
completion_tokens=usage.completion_tokens if usage else 0,
|
|
89
|
+
model=response.model,
|
|
90
|
+
raw={
|
|
91
|
+
"provider": "groq",
|
|
92
|
+
"finish_reason": choice.finish_reason,
|
|
93
|
+
"id": response.id,
|
|
94
|
+
},
|
|
95
|
+
)
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
"""Deterministic mock provider for tests and offline development.
|
|
2
|
+
|
|
3
|
+
``MockProvider`` can be used in two modes:
|
|
4
|
+
|
|
5
|
+
- **Scripted sequence** — pass ``responses=[...]`` to get the given texts back
|
|
6
|
+
in order. After the list is exhausted, returns ``default``.
|
|
7
|
+
- **Single default** — pass only ``default="..."`` to always return the same
|
|
8
|
+
string.
|
|
9
|
+
|
|
10
|
+
Every call is recorded in ``provider.calls`` for inspection in tests.
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
from typing import ClassVar
|
|
16
|
+
|
|
17
|
+
from compactbench.providers.base import CompletionRequest, CompletionResponse, Provider
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
class MockProvider(Provider):
|
|
21
|
+
"""Provider that returns canned responses without touching the network."""
|
|
22
|
+
|
|
23
|
+
key: ClassVar[str] = "mock"
|
|
24
|
+
|
|
25
|
+
def __init__(
|
|
26
|
+
self,
|
|
27
|
+
responses: list[str] | None = None,
|
|
28
|
+
default: str = "",
|
|
29
|
+
) -> None:
|
|
30
|
+
self._responses: list[str] = list(responses or [])
|
|
31
|
+
self._default: str = default
|
|
32
|
+
self._index: int = 0
|
|
33
|
+
self.calls: list[CompletionRequest] = []
|
|
34
|
+
|
|
35
|
+
async def complete(self, request: CompletionRequest) -> CompletionResponse:
|
|
36
|
+
self.calls.append(request)
|
|
37
|
+
if self._index < len(self._responses):
|
|
38
|
+
text = self._responses[self._index]
|
|
39
|
+
self._index += 1
|
|
40
|
+
else:
|
|
41
|
+
text = self._default
|
|
42
|
+
return CompletionResponse(
|
|
43
|
+
text=text,
|
|
44
|
+
prompt_tokens=max(1, len(request.prompt) // 4),
|
|
45
|
+
completion_tokens=max(1, len(text) // 4),
|
|
46
|
+
model=request.model,
|
|
47
|
+
raw={"provider": "mock", "call_index": len(self.calls) - 1},
|
|
48
|
+
)
|
|
49
|
+
|
|
50
|
+
def reset(self) -> None:
|
|
51
|
+
"""Reset the scripted-response pointer and clear recorded calls."""
|
|
52
|
+
self._index = 0
|
|
53
|
+
self.calls.clear()
|
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
"""Ollama local provider.
|
|
2
|
+
|
|
3
|
+
Requires ``compactbench[providers]`` (installs the ``ollama`` SDK). Connects to
|
|
4
|
+
a local or remote Ollama server via ``COMPACTBENCH_OLLAMA_BASE_URL`` (default
|
|
5
|
+
``http://localhost:11434``).
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import os
|
|
11
|
+
from typing import Any, ClassVar
|
|
12
|
+
|
|
13
|
+
from compactbench.providers._retry import retry_with_backoff
|
|
14
|
+
from compactbench.providers.base import (
|
|
15
|
+
CompletionRequest,
|
|
16
|
+
CompletionResponse,
|
|
17
|
+
Provider,
|
|
18
|
+
)
|
|
19
|
+
from compactbench.providers.errors import ProviderError
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class OllamaProvider(Provider):
|
|
23
|
+
"""Async Ollama client (runs models locally)."""
|
|
24
|
+
|
|
25
|
+
key: ClassVar[str] = "ollama"
|
|
26
|
+
|
|
27
|
+
def __init__(
|
|
28
|
+
self,
|
|
29
|
+
*,
|
|
30
|
+
base_url: str | None = None,
|
|
31
|
+
max_retries: int = 3,
|
|
32
|
+
base_backoff_seconds: float = 1.0,
|
|
33
|
+
) -> None:
|
|
34
|
+
try:
|
|
35
|
+
from ollama import AsyncClient
|
|
36
|
+
except ImportError as exc:
|
|
37
|
+
raise ProviderError(
|
|
38
|
+
"ollama SDK is not installed. Install with: pip install 'compactbench[providers]'"
|
|
39
|
+
) from exc
|
|
40
|
+
|
|
41
|
+
resolved_url = base_url or os.environ.get(
|
|
42
|
+
"COMPACTBENCH_OLLAMA_BASE_URL", "http://localhost:11434"
|
|
43
|
+
)
|
|
44
|
+
self._client: Any = AsyncClient(host=resolved_url)
|
|
45
|
+
self._max_retries = max_retries
|
|
46
|
+
self._base_backoff_seconds = base_backoff_seconds
|
|
47
|
+
|
|
48
|
+
async def complete(self, request: CompletionRequest) -> CompletionResponse:
|
|
49
|
+
import httpx
|
|
50
|
+
from ollama import ResponseError
|
|
51
|
+
|
|
52
|
+
def _is_retryable(exc: Exception) -> bool:
|
|
53
|
+
if isinstance(exc, httpx.TimeoutException | httpx.ConnectError):
|
|
54
|
+
return True
|
|
55
|
+
if isinstance(exc, ResponseError):
|
|
56
|
+
code = getattr(exc, "status_code", None)
|
|
57
|
+
return isinstance(code, int) and (code == 429 or code >= 500)
|
|
58
|
+
return False
|
|
59
|
+
|
|
60
|
+
messages: list[dict[str, str]] = []
|
|
61
|
+
if request.system:
|
|
62
|
+
messages.append({"role": "system", "content": request.system})
|
|
63
|
+
messages.append({"role": "user", "content": request.prompt})
|
|
64
|
+
|
|
65
|
+
chat_kwargs: dict[str, Any] = {
|
|
66
|
+
"model": request.model,
|
|
67
|
+
"messages": messages,
|
|
68
|
+
"options": {
|
|
69
|
+
"temperature": request.temperature,
|
|
70
|
+
"num_predict": request.max_tokens,
|
|
71
|
+
},
|
|
72
|
+
}
|
|
73
|
+
if request.response_format and request.response_format.get("type") == "json_object":
|
|
74
|
+
chat_kwargs["format"] = "json"
|
|
75
|
+
|
|
76
|
+
async def _call() -> Any:
|
|
77
|
+
return await self._client.chat(**chat_kwargs)
|
|
78
|
+
|
|
79
|
+
response = await retry_with_backoff(
|
|
80
|
+
_call,
|
|
81
|
+
is_retryable=_is_retryable,
|
|
82
|
+
max_retries=self._max_retries,
|
|
83
|
+
base_delay=self._base_backoff_seconds,
|
|
84
|
+
)
|
|
85
|
+
|
|
86
|
+
message = response.message
|
|
87
|
+
text = getattr(message, "content", None) or ""
|
|
88
|
+
prompt_tokens = getattr(response, "prompt_eval_count", 0) or 0
|
|
89
|
+
completion_tokens = getattr(response, "eval_count", 0) or 0
|
|
90
|
+
|
|
91
|
+
return CompletionResponse(
|
|
92
|
+
text=text,
|
|
93
|
+
prompt_tokens=prompt_tokens,
|
|
94
|
+
completion_tokens=completion_tokens,
|
|
95
|
+
model=request.model,
|
|
96
|
+
raw={
|
|
97
|
+
"provider": "ollama",
|
|
98
|
+
"done_reason": getattr(response, "done_reason", None),
|
|
99
|
+
},
|
|
100
|
+
)
|
compactbench/py.typed
ADDED
|
File without changes
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
"""Run orchestration: suite → cases → drift cycles → scored results.
|
|
2
|
+
|
|
3
|
+
Implements WO-007. The public surface is :func:`run_experiment` plus
|
|
4
|
+
persistence helpers; the CLI wires these together.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from compactbench.runner._resolver import resolve_compactor_class
|
|
8
|
+
from compactbench.runner.continuation import extend_with_continuation
|
|
9
|
+
from compactbench.runner.cycle import CycleExecutionResult, execute_cycle
|
|
10
|
+
from compactbench.runner.errors import MethodResolutionError, ResumeError, RunnerError
|
|
11
|
+
from compactbench.runner.evaluation import (
|
|
12
|
+
build_evaluation_prompt,
|
|
13
|
+
evaluate_items,
|
|
14
|
+
render_artifact_for_prompt,
|
|
15
|
+
)
|
|
16
|
+
from compactbench.runner.persistence import (
|
|
17
|
+
SCORER_VERSION,
|
|
18
|
+
CaseCompleteEvent,
|
|
19
|
+
ResultsWriter,
|
|
20
|
+
RunEndEvent,
|
|
21
|
+
RunStartEvent,
|
|
22
|
+
aggregate_run_metrics,
|
|
23
|
+
completed_case_ids,
|
|
24
|
+
iter_events,
|
|
25
|
+
read_run_start,
|
|
26
|
+
to_run_result,
|
|
27
|
+
)
|
|
28
|
+
from compactbench.runner.run import RunArgs, run_experiment
|
|
29
|
+
|
|
30
|
+
__all__ = [
|
|
31
|
+
"SCORER_VERSION",
|
|
32
|
+
"CaseCompleteEvent",
|
|
33
|
+
"CycleExecutionResult",
|
|
34
|
+
"MethodResolutionError",
|
|
35
|
+
"ResultsWriter",
|
|
36
|
+
"ResumeError",
|
|
37
|
+
"RunArgs",
|
|
38
|
+
"RunEndEvent",
|
|
39
|
+
"RunStartEvent",
|
|
40
|
+
"RunnerError",
|
|
41
|
+
"aggregate_run_metrics",
|
|
42
|
+
"build_evaluation_prompt",
|
|
43
|
+
"completed_case_ids",
|
|
44
|
+
"evaluate_items",
|
|
45
|
+
"execute_cycle",
|
|
46
|
+
"extend_with_continuation",
|
|
47
|
+
"iter_events",
|
|
48
|
+
"read_run_start",
|
|
49
|
+
"render_artifact_for_prompt",
|
|
50
|
+
"resolve_compactor_class",
|
|
51
|
+
"run_experiment",
|
|
52
|
+
"to_run_result",
|
|
53
|
+
]
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
"""Resolve ``--method`` CLI specs to Compactor classes."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import importlib.util
|
|
6
|
+
import sys
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
|
|
9
|
+
from compactbench.compactors import Compactor, get_built_in
|
|
10
|
+
from compactbench.compactors.errors import UnknownCompactorError
|
|
11
|
+
from compactbench.runner.errors import MethodResolutionError
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def resolve_compactor_class(spec: str) -> type[Compactor]:
|
|
15
|
+
"""Resolve a ``--method`` spec to a :class:`Compactor` class.
|
|
16
|
+
|
|
17
|
+
Two spec forms are supported:
|
|
18
|
+
|
|
19
|
+
- ``built-in:<key>`` — look up in the built-in registry.
|
|
20
|
+
- ``<path>:<ClassName>`` — load ``ClassName`` from the Python file at ``<path>``.
|
|
21
|
+
The class must subclass :class:`Compactor`.
|
|
22
|
+
"""
|
|
23
|
+
if spec.startswith("built-in:"):
|
|
24
|
+
key = spec[len("built-in:") :]
|
|
25
|
+
try:
|
|
26
|
+
return get_built_in(key)
|
|
27
|
+
except UnknownCompactorError as exc:
|
|
28
|
+
raise MethodResolutionError(str(exc)) from exc
|
|
29
|
+
|
|
30
|
+
if ":" not in spec:
|
|
31
|
+
raise MethodResolutionError(
|
|
32
|
+
f"could not resolve method {spec!r}. Use 'built-in:<key>' or '<path>:<ClassName>'."
|
|
33
|
+
)
|
|
34
|
+
|
|
35
|
+
path_str, class_name = spec.rsplit(":", 1)
|
|
36
|
+
path = Path(path_str)
|
|
37
|
+
if not path.is_file():
|
|
38
|
+
raise MethodResolutionError(f"method file not found: {path}")
|
|
39
|
+
if path.suffix != ".py":
|
|
40
|
+
raise MethodResolutionError(f"method file must be a .py file: {path}")
|
|
41
|
+
|
|
42
|
+
module_spec = importlib.util.spec_from_file_location(f"_user_compactor_{path.stem}", path)
|
|
43
|
+
if module_spec is None or module_spec.loader is None:
|
|
44
|
+
raise MethodResolutionError(f"could not load module from {path}")
|
|
45
|
+
module = importlib.util.module_from_spec(module_spec)
|
|
46
|
+
sys.modules[module_spec.name] = module
|
|
47
|
+
try:
|
|
48
|
+
module_spec.loader.exec_module(module)
|
|
49
|
+
except Exception as exc:
|
|
50
|
+
raise MethodResolutionError(f"error importing {path}: {exc}") from exc
|
|
51
|
+
|
|
52
|
+
cls = getattr(module, class_name, None)
|
|
53
|
+
if cls is None:
|
|
54
|
+
raise MethodResolutionError(f"class {class_name!r} not found in {path}")
|
|
55
|
+
if not isinstance(cls, type) or not issubclass(cls, Compactor):
|
|
56
|
+
raise MethodResolutionError(f"{class_name!r} in {path} is not a subclass of Compactor")
|
|
57
|
+
return cls
|