compactbench 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (66) hide show
  1. compactbench/__init__.py +10 -0
  2. compactbench/__main__.py +6 -0
  3. compactbench/cli.py +343 -0
  4. compactbench/compactors/__init__.py +47 -0
  5. compactbench/compactors/_state_parser.py +118 -0
  6. compactbench/compactors/_utils.py +37 -0
  7. compactbench/compactors/base.py +38 -0
  8. compactbench/compactors/errors.py +11 -0
  9. compactbench/compactors/hierarchical_summary.py +113 -0
  10. compactbench/compactors/hybrid_ledger.py +115 -0
  11. compactbench/compactors/naive_summary.py +54 -0
  12. compactbench/compactors/structured_state.py +64 -0
  13. compactbench/config.py +36 -0
  14. compactbench/contracts/__init__.py +41 -0
  15. compactbench/contracts/artifact.py +49 -0
  16. compactbench/contracts/case.py +81 -0
  17. compactbench/contracts/result.py +79 -0
  18. compactbench/dsl/__init__.py +66 -0
  19. compactbench/dsl/errors.py +23 -0
  20. compactbench/dsl/generators.py +243 -0
  21. compactbench/dsl/models.py +107 -0
  22. compactbench/dsl/parser.py +55 -0
  23. compactbench/dsl/substitution.py +32 -0
  24. compactbench/dsl/validator.py +97 -0
  25. compactbench/engine/__init__.py +29 -0
  26. compactbench/engine/difficulty.py +54 -0
  27. compactbench/engine/distractors.py +69 -0
  28. compactbench/engine/errors.py +11 -0
  29. compactbench/engine/generation.py +96 -0
  30. compactbench/engine/seeds.py +31 -0
  31. compactbench/engine/transcript.py +84 -0
  32. compactbench/leaderboard/__init__.py +38 -0
  33. compactbench/leaderboard/errors.py +11 -0
  34. compactbench/leaderboard/projection.py +103 -0
  35. compactbench/leaderboard/qualification.py +90 -0
  36. compactbench/leaderboard/ranking.py +71 -0
  37. compactbench/providers/__init__.py +52 -0
  38. compactbench/providers/_retry.py +48 -0
  39. compactbench/providers/base.py +44 -0
  40. compactbench/providers/errors.py +15 -0
  41. compactbench/providers/google_ai_studio.py +112 -0
  42. compactbench/providers/groq.py +95 -0
  43. compactbench/providers/mock.py +53 -0
  44. compactbench/providers/ollama.py +100 -0
  45. compactbench/py.typed +0 -0
  46. compactbench/runner/__init__.py +53 -0
  47. compactbench/runner/_resolver.py +57 -0
  48. compactbench/runner/continuation.py +78 -0
  49. compactbench/runner/cycle.py +67 -0
  50. compactbench/runner/errors.py +15 -0
  51. compactbench/runner/evaluation.py +71 -0
  52. compactbench/runner/persistence.py +227 -0
  53. compactbench/runner/run.py +257 -0
  54. compactbench/scoring/__init__.py +48 -0
  55. compactbench/scoring/checks.py +92 -0
  56. compactbench/scoring/compression.py +66 -0
  57. compactbench/scoring/contradictions.py +70 -0
  58. compactbench/scoring/drift.py +30 -0
  59. compactbench/scoring/errors.py +7 -0
  60. compactbench/scoring/scorer.py +77 -0
  61. compactbench/utils/__init__.py +1 -0
  62. compactbench-0.1.0.dist-info/METADATA +161 -0
  63. compactbench-0.1.0.dist-info/RECORD +66 -0
  64. compactbench-0.1.0.dist-info/WHEEL +4 -0
  65. compactbench-0.1.0.dist-info/entry_points.txt +2 -0
  66. compactbench-0.1.0.dist-info/licenses/LICENSE +201 -0
@@ -0,0 +1,52 @@
1
+ """Model provider abstractions and concrete clients."""
2
+
3
+ from compactbench.providers.base import (
4
+ CompletionRequest,
5
+ CompletionResponse,
6
+ Provider,
7
+ )
8
+ from compactbench.providers.errors import (
9
+ ProviderError,
10
+ ProviderResponseError,
11
+ UnknownProviderError,
12
+ )
13
+ from compactbench.providers.google_ai_studio import GoogleAIStudioProvider
14
+ from compactbench.providers.groq import GroqProvider
15
+ from compactbench.providers.mock import MockProvider
16
+ from compactbench.providers.ollama import OllamaProvider
17
+
18
+ _REGISTRY: dict[str, type[Provider]] = {
19
+ MockProvider.key: MockProvider,
20
+ GroqProvider.key: GroqProvider,
21
+ GoogleAIStudioProvider.key: GoogleAIStudioProvider,
22
+ OllamaProvider.key: OllamaProvider,
23
+ }
24
+
25
+
26
+ def list_providers() -> list[str]:
27
+ """Return the sorted list of registered provider keys."""
28
+ return sorted(_REGISTRY)
29
+
30
+
31
+ def get_provider_cls(key: str) -> type[Provider]:
32
+ """Return the provider class for ``key`` or raise :class:`UnknownProviderError`."""
33
+ cls = _REGISTRY.get(key)
34
+ if cls is None:
35
+ raise UnknownProviderError(f"unknown provider {key!r}. Known: {sorted(_REGISTRY)}")
36
+ return cls
37
+
38
+
39
+ __all__ = [
40
+ "CompletionRequest",
41
+ "CompletionResponse",
42
+ "GoogleAIStudioProvider",
43
+ "GroqProvider",
44
+ "MockProvider",
45
+ "OllamaProvider",
46
+ "Provider",
47
+ "ProviderError",
48
+ "ProviderResponseError",
49
+ "UnknownProviderError",
50
+ "get_provider_cls",
51
+ "list_providers",
52
+ ]
@@ -0,0 +1,48 @@
1
+ """Shared async exponential-backoff retry helper for providers.
2
+
3
+ Each provider decides which of its SDK's exceptions are retryable (typically
4
+ rate-limit and transient network errors) and hands a predicate plus the async
5
+ call to :func:`retry_with_backoff`.
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ import asyncio
11
+ from collections.abc import Awaitable, Callable
12
+ from typing import TypeVar
13
+
14
+ T = TypeVar("T")
15
+
16
+
17
+ async def retry_with_backoff(
18
+ operation: Callable[[], Awaitable[T]],
19
+ *,
20
+ is_retryable: Callable[[Exception], bool],
21
+ max_retries: int = 3,
22
+ base_delay: float = 2.0,
23
+ max_delay: float = 30.0,
24
+ ) -> T:
25
+ """Call ``operation`` with exponential backoff on retryable exceptions.
26
+
27
+ Returns the operation's result on first success. On a retryable failure,
28
+ waits ``min(base_delay * 2**attempt, max_delay)`` seconds and retries.
29
+ After ``max_retries`` failed attempts the last exception is re-raised.
30
+ Non-retryable exceptions are re-raised immediately.
31
+ """
32
+ if max_retries < 1:
33
+ raise ValueError(f"max_retries must be >= 1, got {max_retries}")
34
+
35
+ last_exc: Exception | None = None
36
+ for attempt in range(max_retries):
37
+ try:
38
+ return await operation()
39
+ except Exception as exc:
40
+ if not is_retryable(exc):
41
+ raise
42
+ last_exc = exc
43
+ if attempt < max_retries - 1:
44
+ delay = min(base_delay * (2**attempt), max_delay)
45
+ await asyncio.sleep(delay)
46
+
47
+ assert last_exc is not None # unreachable unless last_exc was set in the loop
48
+ raise last_exc
@@ -0,0 +1,44 @@
1
+ """Model provider ABC and request/response shapes.
2
+
3
+ Concrete implementations (Groq, Google AI Studio, Ollama) live in WO-006.
4
+ The mock provider below is part of WO-005 so compactors can be tested offline.
5
+ """
6
+
7
+ from __future__ import annotations
8
+
9
+ from abc import ABC, abstractmethod
10
+ from dataclasses import dataclass, field
11
+ from typing import Any, ClassVar
12
+
13
+
14
+ @dataclass(frozen=True)
15
+ class CompletionRequest:
16
+ """A normalized request passed to a provider's ``complete`` method."""
17
+
18
+ model: str
19
+ prompt: str
20
+ system: str | None = None
21
+ max_tokens: int = 2048
22
+ temperature: float = 0.0
23
+ response_format: dict[str, Any] | None = None
24
+
25
+
26
+ @dataclass(frozen=True)
27
+ class CompletionResponse:
28
+ """A normalized response returned from a provider."""
29
+
30
+ text: str
31
+ prompt_tokens: int
32
+ completion_tokens: int
33
+ model: str
34
+ raw: dict[str, Any] = field(default_factory=dict[str, Any])
35
+
36
+
37
+ class Provider(ABC):
38
+ """Base class every provider must implement."""
39
+
40
+ key: ClassVar[str]
41
+
42
+ @abstractmethod
43
+ async def complete(self, request: CompletionRequest) -> CompletionResponse:
44
+ """Issue a completion request and return a normalized response."""
@@ -0,0 +1,15 @@
1
+ """Provider-specific exceptions."""
2
+
3
+ from __future__ import annotations
4
+
5
+
6
+ class ProviderError(Exception):
7
+ """Base exception for provider operations."""
8
+
9
+
10
+ class UnknownProviderError(ProviderError):
11
+ """Requested provider key is not registered."""
12
+
13
+
14
+ class ProviderResponseError(ProviderError):
15
+ """Provider returned a malformed or unusable response."""
@@ -0,0 +1,112 @@
1
+ """Google AI Studio provider (Gemini 2.0 Flash, etc.).
2
+
3
+ Requires ``compactbench[providers]`` (installs the ``google-genai`` SDK) and a
4
+ ``COMPACTBENCH_GOOGLE_AI_STUDIO_API_KEY`` environment variable.
5
+ """
6
+
7
+ from __future__ import annotations
8
+
9
+ import os
10
+ from typing import Any, ClassVar, cast
11
+
12
+ from compactbench.providers._retry import retry_with_backoff
13
+ from compactbench.providers.base import (
14
+ CompletionRequest,
15
+ CompletionResponse,
16
+ Provider,
17
+ )
18
+ from compactbench.providers.errors import ProviderError
19
+
20
+
21
+ class GoogleAIStudioProvider(Provider):
22
+ """Async Google AI Studio client (via the google-genai SDK)."""
23
+
24
+ key: ClassVar[str] = "google-ai-studio"
25
+
26
+ def __init__(
27
+ self,
28
+ *,
29
+ api_key: str | None = None,
30
+ max_retries: int = 3,
31
+ base_backoff_seconds: float = 2.0,
32
+ ) -> None:
33
+ try:
34
+ from google import genai
35
+ except ImportError as exc:
36
+ raise ProviderError(
37
+ "google-genai SDK is not installed. "
38
+ "Install with: pip install 'compactbench[providers]'"
39
+ ) from exc
40
+
41
+ resolved_key = api_key or os.environ.get("COMPACTBENCH_GOOGLE_AI_STUDIO_API_KEY")
42
+ if not resolved_key:
43
+ raise ProviderError(
44
+ "Google AI Studio API key required. Set "
45
+ "COMPACTBENCH_GOOGLE_AI_STUDIO_API_KEY or pass api_key=."
46
+ )
47
+
48
+ self._client: Any = genai.Client(api_key=resolved_key)
49
+ self._max_retries = max_retries
50
+ self._base_backoff_seconds = base_backoff_seconds
51
+
52
+ async def complete(self, request: CompletionRequest) -> CompletionResponse:
53
+ from google.genai import errors as genai_errors
54
+ from google.genai import types
55
+
56
+ def _is_retryable(exc: Exception) -> bool:
57
+ if not isinstance(exc, genai_errors.APIError):
58
+ return False
59
+ code = getattr(exc, "code", None)
60
+ if code is None:
61
+ code = getattr(exc, "status_code", None)
62
+ if not isinstance(code, int):
63
+ return False
64
+ return code == 429 or code >= 500
65
+
66
+ config_kwargs: dict[str, Any] = {
67
+ "temperature": request.temperature,
68
+ "max_output_tokens": request.max_tokens,
69
+ }
70
+ if request.system:
71
+ config_kwargs["system_instruction"] = request.system
72
+ if request.response_format and request.response_format.get("type") == "json_object":
73
+ config_kwargs["response_mime_type"] = "application/json"
74
+ config = types.GenerateContentConfig(**config_kwargs)
75
+
76
+ async def _call() -> Any:
77
+ return await self._client.aio.models.generate_content(
78
+ model=request.model,
79
+ contents=request.prompt,
80
+ config=config,
81
+ )
82
+
83
+ response = await retry_with_backoff(
84
+ _call,
85
+ is_retryable=_is_retryable,
86
+ max_retries=self._max_retries,
87
+ base_delay=self._base_backoff_seconds,
88
+ )
89
+
90
+ text = getattr(response, "text", None) or ""
91
+ usage_metadata = getattr(response, "usage_metadata", None)
92
+ prompt_tokens = (
93
+ (getattr(usage_metadata, "prompt_token_count", 0) or 0) if usage_metadata else 0
94
+ )
95
+ completion_tokens = (
96
+ (getattr(usage_metadata, "candidates_token_count", 0) or 0) if usage_metadata else 0
97
+ )
98
+ raw_candidates = getattr(response, "candidates", None)
99
+ candidates: list[Any] = (
100
+ cast("list[Any]", raw_candidates) if isinstance(raw_candidates, list) else []
101
+ )
102
+ finish_reason = getattr(candidates[0], "finish_reason", None) if candidates else None
103
+ return CompletionResponse(
104
+ text=text,
105
+ prompt_tokens=prompt_tokens,
106
+ completion_tokens=completion_tokens,
107
+ model=request.model,
108
+ raw={
109
+ "provider": "google-ai-studio",
110
+ "finish_reason": str(finish_reason) if finish_reason is not None else None,
111
+ },
112
+ )
@@ -0,0 +1,95 @@
1
+ """Groq Cloud provider (Llama 3.3 70B, Kimi K2, etc.).
2
+
3
+ Requires ``compactbench[providers]`` (installs the ``groq`` SDK) and a
4
+ ``COMPACTBENCH_GROQ_API_KEY`` environment variable.
5
+ """
6
+
7
+ from __future__ import annotations
8
+
9
+ import os
10
+ from typing import Any, ClassVar
11
+
12
+ from compactbench.providers._retry import retry_with_backoff
13
+ from compactbench.providers.base import (
14
+ CompletionRequest,
15
+ CompletionResponse,
16
+ Provider,
17
+ )
18
+ from compactbench.providers.errors import ProviderError, ProviderResponseError
19
+
20
+
21
+ class GroqProvider(Provider):
22
+ """Async Groq client with rate-limit backoff."""
23
+
24
+ key: ClassVar[str] = "groq"
25
+
26
+ def __init__(
27
+ self,
28
+ *,
29
+ api_key: str | None = None,
30
+ max_retries: int = 3,
31
+ base_backoff_seconds: float = 2.0,
32
+ ) -> None:
33
+ try:
34
+ from groq import AsyncGroq
35
+ except ImportError as exc:
36
+ raise ProviderError(
37
+ "groq SDK is not installed. Install with: pip install 'compactbench[providers]'"
38
+ ) from exc
39
+
40
+ resolved_key = api_key or os.environ.get("COMPACTBENCH_GROQ_API_KEY")
41
+ if not resolved_key:
42
+ raise ProviderError(
43
+ "Groq API key required. Set COMPACTBENCH_GROQ_API_KEY or pass api_key=."
44
+ )
45
+
46
+ self._client: Any = AsyncGroq(api_key=resolved_key)
47
+ self._max_retries = max_retries
48
+ self._base_backoff_seconds = base_backoff_seconds
49
+
50
+ async def complete(self, request: CompletionRequest) -> CompletionResponse:
51
+ from groq import APIConnectionError, APITimeoutError, RateLimitError
52
+
53
+ def _is_retryable(exc: Exception) -> bool:
54
+ return isinstance(exc, RateLimitError | APITimeoutError | APIConnectionError)
55
+
56
+ messages: list[dict[str, str]] = []
57
+ if request.system:
58
+ messages.append({"role": "system", "content": request.system})
59
+ messages.append({"role": "user", "content": request.prompt})
60
+
61
+ kwargs: dict[str, Any] = {
62
+ "model": request.model,
63
+ "messages": messages,
64
+ "temperature": request.temperature,
65
+ "max_tokens": request.max_tokens,
66
+ }
67
+ if request.response_format:
68
+ kwargs["response_format"] = request.response_format
69
+
70
+ async def _call() -> Any:
71
+ return await self._client.chat.completions.create(**kwargs)
72
+
73
+ response = await retry_with_backoff(
74
+ _call,
75
+ is_retryable=_is_retryable,
76
+ max_retries=self._max_retries,
77
+ base_delay=self._base_backoff_seconds,
78
+ )
79
+
80
+ if not response.choices:
81
+ raise ProviderResponseError("Groq returned no choices in response")
82
+ choice = response.choices[0]
83
+ text = choice.message.content or ""
84
+ usage = response.usage
85
+ return CompletionResponse(
86
+ text=text,
87
+ prompt_tokens=usage.prompt_tokens if usage else 0,
88
+ completion_tokens=usage.completion_tokens if usage else 0,
89
+ model=response.model,
90
+ raw={
91
+ "provider": "groq",
92
+ "finish_reason": choice.finish_reason,
93
+ "id": response.id,
94
+ },
95
+ )
@@ -0,0 +1,53 @@
1
+ """Deterministic mock provider for tests and offline development.
2
+
3
+ ``MockProvider`` can be used in two modes:
4
+
5
+ - **Scripted sequence** — pass ``responses=[...]`` to get the given texts back
6
+ in order. After the list is exhausted, returns ``default``.
7
+ - **Single default** — pass only ``default="..."`` to always return the same
8
+ string.
9
+
10
+ Every call is recorded in ``provider.calls`` for inspection in tests.
11
+ """
12
+
13
+ from __future__ import annotations
14
+
15
+ from typing import ClassVar
16
+
17
+ from compactbench.providers.base import CompletionRequest, CompletionResponse, Provider
18
+
19
+
20
+ class MockProvider(Provider):
21
+ """Provider that returns canned responses without touching the network."""
22
+
23
+ key: ClassVar[str] = "mock"
24
+
25
+ def __init__(
26
+ self,
27
+ responses: list[str] | None = None,
28
+ default: str = "",
29
+ ) -> None:
30
+ self._responses: list[str] = list(responses or [])
31
+ self._default: str = default
32
+ self._index: int = 0
33
+ self.calls: list[CompletionRequest] = []
34
+
35
+ async def complete(self, request: CompletionRequest) -> CompletionResponse:
36
+ self.calls.append(request)
37
+ if self._index < len(self._responses):
38
+ text = self._responses[self._index]
39
+ self._index += 1
40
+ else:
41
+ text = self._default
42
+ return CompletionResponse(
43
+ text=text,
44
+ prompt_tokens=max(1, len(request.prompt) // 4),
45
+ completion_tokens=max(1, len(text) // 4),
46
+ model=request.model,
47
+ raw={"provider": "mock", "call_index": len(self.calls) - 1},
48
+ )
49
+
50
+ def reset(self) -> None:
51
+ """Reset the scripted-response pointer and clear recorded calls."""
52
+ self._index = 0
53
+ self.calls.clear()
@@ -0,0 +1,100 @@
1
+ """Ollama local provider.
2
+
3
+ Requires ``compactbench[providers]`` (installs the ``ollama`` SDK). Connects to
4
+ a local or remote Ollama server via ``COMPACTBENCH_OLLAMA_BASE_URL`` (default
5
+ ``http://localhost:11434``).
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ import os
11
+ from typing import Any, ClassVar
12
+
13
+ from compactbench.providers._retry import retry_with_backoff
14
+ from compactbench.providers.base import (
15
+ CompletionRequest,
16
+ CompletionResponse,
17
+ Provider,
18
+ )
19
+ from compactbench.providers.errors import ProviderError
20
+
21
+
22
+ class OllamaProvider(Provider):
23
+ """Async Ollama client (runs models locally)."""
24
+
25
+ key: ClassVar[str] = "ollama"
26
+
27
+ def __init__(
28
+ self,
29
+ *,
30
+ base_url: str | None = None,
31
+ max_retries: int = 3,
32
+ base_backoff_seconds: float = 1.0,
33
+ ) -> None:
34
+ try:
35
+ from ollama import AsyncClient
36
+ except ImportError as exc:
37
+ raise ProviderError(
38
+ "ollama SDK is not installed. Install with: pip install 'compactbench[providers]'"
39
+ ) from exc
40
+
41
+ resolved_url = base_url or os.environ.get(
42
+ "COMPACTBENCH_OLLAMA_BASE_URL", "http://localhost:11434"
43
+ )
44
+ self._client: Any = AsyncClient(host=resolved_url)
45
+ self._max_retries = max_retries
46
+ self._base_backoff_seconds = base_backoff_seconds
47
+
48
+ async def complete(self, request: CompletionRequest) -> CompletionResponse:
49
+ import httpx
50
+ from ollama import ResponseError
51
+
52
+ def _is_retryable(exc: Exception) -> bool:
53
+ if isinstance(exc, httpx.TimeoutException | httpx.ConnectError):
54
+ return True
55
+ if isinstance(exc, ResponseError):
56
+ code = getattr(exc, "status_code", None)
57
+ return isinstance(code, int) and (code == 429 or code >= 500)
58
+ return False
59
+
60
+ messages: list[dict[str, str]] = []
61
+ if request.system:
62
+ messages.append({"role": "system", "content": request.system})
63
+ messages.append({"role": "user", "content": request.prompt})
64
+
65
+ chat_kwargs: dict[str, Any] = {
66
+ "model": request.model,
67
+ "messages": messages,
68
+ "options": {
69
+ "temperature": request.temperature,
70
+ "num_predict": request.max_tokens,
71
+ },
72
+ }
73
+ if request.response_format and request.response_format.get("type") == "json_object":
74
+ chat_kwargs["format"] = "json"
75
+
76
+ async def _call() -> Any:
77
+ return await self._client.chat(**chat_kwargs)
78
+
79
+ response = await retry_with_backoff(
80
+ _call,
81
+ is_retryable=_is_retryable,
82
+ max_retries=self._max_retries,
83
+ base_delay=self._base_backoff_seconds,
84
+ )
85
+
86
+ message = response.message
87
+ text = getattr(message, "content", None) or ""
88
+ prompt_tokens = getattr(response, "prompt_eval_count", 0) or 0
89
+ completion_tokens = getattr(response, "eval_count", 0) or 0
90
+
91
+ return CompletionResponse(
92
+ text=text,
93
+ prompt_tokens=prompt_tokens,
94
+ completion_tokens=completion_tokens,
95
+ model=request.model,
96
+ raw={
97
+ "provider": "ollama",
98
+ "done_reason": getattr(response, "done_reason", None),
99
+ },
100
+ )
compactbench/py.typed ADDED
File without changes
@@ -0,0 +1,53 @@
1
+ """Run orchestration: suite → cases → drift cycles → scored results.
2
+
3
+ Implements WO-007. The public surface is :func:`run_experiment` plus
4
+ persistence helpers; the CLI wires these together.
5
+ """
6
+
7
+ from compactbench.runner._resolver import resolve_compactor_class
8
+ from compactbench.runner.continuation import extend_with_continuation
9
+ from compactbench.runner.cycle import CycleExecutionResult, execute_cycle
10
+ from compactbench.runner.errors import MethodResolutionError, ResumeError, RunnerError
11
+ from compactbench.runner.evaluation import (
12
+ build_evaluation_prompt,
13
+ evaluate_items,
14
+ render_artifact_for_prompt,
15
+ )
16
+ from compactbench.runner.persistence import (
17
+ SCORER_VERSION,
18
+ CaseCompleteEvent,
19
+ ResultsWriter,
20
+ RunEndEvent,
21
+ RunStartEvent,
22
+ aggregate_run_metrics,
23
+ completed_case_ids,
24
+ iter_events,
25
+ read_run_start,
26
+ to_run_result,
27
+ )
28
+ from compactbench.runner.run import RunArgs, run_experiment
29
+
30
+ __all__ = [
31
+ "SCORER_VERSION",
32
+ "CaseCompleteEvent",
33
+ "CycleExecutionResult",
34
+ "MethodResolutionError",
35
+ "ResultsWriter",
36
+ "ResumeError",
37
+ "RunArgs",
38
+ "RunEndEvent",
39
+ "RunStartEvent",
40
+ "RunnerError",
41
+ "aggregate_run_metrics",
42
+ "build_evaluation_prompt",
43
+ "completed_case_ids",
44
+ "evaluate_items",
45
+ "execute_cycle",
46
+ "extend_with_continuation",
47
+ "iter_events",
48
+ "read_run_start",
49
+ "render_artifact_for_prompt",
50
+ "resolve_compactor_class",
51
+ "run_experiment",
52
+ "to_run_result",
53
+ ]
@@ -0,0 +1,57 @@
1
+ """Resolve ``--method`` CLI specs to Compactor classes."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import importlib.util
6
+ import sys
7
+ from pathlib import Path
8
+
9
+ from compactbench.compactors import Compactor, get_built_in
10
+ from compactbench.compactors.errors import UnknownCompactorError
11
+ from compactbench.runner.errors import MethodResolutionError
12
+
13
+
14
+ def resolve_compactor_class(spec: str) -> type[Compactor]:
15
+ """Resolve a ``--method`` spec to a :class:`Compactor` class.
16
+
17
+ Two spec forms are supported:
18
+
19
+ - ``built-in:<key>`` — look up in the built-in registry.
20
+ - ``<path>:<ClassName>`` — load ``ClassName`` from the Python file at ``<path>``.
21
+ The class must subclass :class:`Compactor`.
22
+ """
23
+ if spec.startswith("built-in:"):
24
+ key = spec[len("built-in:") :]
25
+ try:
26
+ return get_built_in(key)
27
+ except UnknownCompactorError as exc:
28
+ raise MethodResolutionError(str(exc)) from exc
29
+
30
+ if ":" not in spec:
31
+ raise MethodResolutionError(
32
+ f"could not resolve method {spec!r}. Use 'built-in:<key>' or '<path>:<ClassName>'."
33
+ )
34
+
35
+ path_str, class_name = spec.rsplit(":", 1)
36
+ path = Path(path_str)
37
+ if not path.is_file():
38
+ raise MethodResolutionError(f"method file not found: {path}")
39
+ if path.suffix != ".py":
40
+ raise MethodResolutionError(f"method file must be a .py file: {path}")
41
+
42
+ module_spec = importlib.util.spec_from_file_location(f"_user_compactor_{path.stem}", path)
43
+ if module_spec is None or module_spec.loader is None:
44
+ raise MethodResolutionError(f"could not load module from {path}")
45
+ module = importlib.util.module_from_spec(module_spec)
46
+ sys.modules[module_spec.name] = module
47
+ try:
48
+ module_spec.loader.exec_module(module)
49
+ except Exception as exc:
50
+ raise MethodResolutionError(f"error importing {path}: {exc}") from exc
51
+
52
+ cls = getattr(module, class_name, None)
53
+ if cls is None:
54
+ raise MethodResolutionError(f"class {class_name!r} not found in {path}")
55
+ if not isinstance(cls, type) or not issubclass(cls, Compactor):
56
+ raise MethodResolutionError(f"{class_name!r} in {path} is not a subclass of Compactor")
57
+ return cls