strop-cli 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- strop/__init__.py +5 -0
- strop/__main__.py +5 -0
- strop/analysis/__init__.py +17 -0
- strop/analysis/classify.py +52 -0
- strop/analysis/evidence.py +52 -0
- strop/analysis/facts.py +74 -0
- strop/analysis/patterns.py +90 -0
- strop/cli/__init__.py +5 -0
- strop/cli/analyze.py +93 -0
- strop/cli/app.py +82 -0
- strop/cli/data.py +111 -0
- strop/cli/evaluate.py +145 -0
- strop/cli/export.py +74 -0
- strop/cli/final.py +106 -0
- strop/cli/init.py +131 -0
- strop/cli/optimize.py +156 -0
- strop/cli/options.py +20 -0
- strop/cli/progress.py +18 -0
- strop/cli/prompts.py +110 -0
- strop/cli/status.py +68 -0
- strop/cli/versions.py +123 -0
- strop/config/__init__.py +16 -0
- strop/config/base.py +9 -0
- strop/config/constraints.py +33 -0
- strop/config/dataset.py +63 -0
- strop/config/enums.py +52 -0
- strop/config/io.py +52 -0
- strop/config/limits.py +19 -0
- strop/config/llm.py +42 -0
- strop/config/metrics.py +86 -0
- strop/config/optimization.py +23 -0
- strop/config/output.py +31 -0
- strop/config/overrides.py +22 -0
- strop/config/root.py +36 -0
- strop/config/rules.py +67 -0
- strop/config/split.py +23 -0
- strop/data/__init__.py +58 -0
- strop/data/loaders.py +85 -0
- strop/data/models.py +44 -0
- strop/data/rows.py +99 -0
- strop/data/splitting.py +103 -0
- strop/env.py +44 -0
- strop/errors.py +59 -0
- strop/evaluation/__init__.py +1 -0
- strop/evaluation/budget.py +67 -0
- strop/evaluation/budgeted.py +23 -0
- strop/evaluation/examples.py +49 -0
- strop/evaluation/models.py +105 -0
- strop/evaluation/parsing.py +96 -0
- strop/evaluation/runner.py +107 -0
- strop/formatting.py +14 -0
- strop/llm/__init__.py +7 -0
- strop/llm/caching.py +28 -0
- strop/llm/client.py +15 -0
- strop/llm/factory.py +14 -0
- strop/llm/litellm_client.py +90 -0
- strop/llm/mock.py +43 -0
- strop/llm/retry.py +44 -0
- strop/llm/structured.py +52 -0
- strop/llm/types.py +59 -0
- strop/log.py +49 -0
- strop/metrics/__init__.py +1 -0
- strop/metrics/basic.py +30 -0
- strop/metrics/classification.py +164 -0
- strop/metrics/judge.py +130 -0
- strop/metrics/python_metric.py +101 -0
- strop/metrics/python_runner.py +65 -0
- strop/metrics/registry.py +115 -0
- strop/metrics/rules.py +55 -0
- strop/metrics/structured.py +109 -0
- strop/metrics/suite.py +109 -0
- strop/metrics/text.py +55 -0
- strop/metrics/types.py +106 -0
- strop/optimization/__init__.py +25 -0
- strop/optimization/acceptance.py +170 -0
- strop/optimization/generator.py +92 -0
- strop/optimization/meta_prompt.py +101 -0
- strop/optimization/types.py +69 -0
- strop/prompts/__init__.py +18 -0
- strop/prompts/diff.py +139 -0
- strop/prompts/files.py +92 -0
- strop/prompts/models.py +51 -0
- strop/prompts/render.py +16 -0
- strop/prompts/template.py +45 -0
- strop/prompts/validation.py +31 -0
- strop/prompts/versions.py +34 -0
- strop/py.typed +0 -0
- strop/reporting/__init__.py +6 -0
- strop/reporting/console.py +20 -0
- strop/reporting/diff_view.py +102 -0
- strop/reporting/evaluation_view.py +141 -0
- strop/reporting/final_view.py +109 -0
- strop/reporting/prompt_export.py +58 -0
- strop/reporting/report_markdown.py +157 -0
- strop/reporting/run_view.py +118 -0
- strop/services/__init__.py +1 -0
- strop/services/analysis.py +124 -0
- strop/services/clean.py +38 -0
- strop/services/compare.py +60 -0
- strop/services/context.py +70 -0
- strop/services/data.py +65 -0
- strop/services/evaluate.py +105 -0
- strop/services/evaluator.py +169 -0
- strop/services/export.py +90 -0
- strop/services/final_test.py +69 -0
- strop/services/metrics.py +89 -0
- strop/services/optimize.py +187 -0
- strop/services/partitions.py +46 -0
- strop/services/project.py +55 -0
- strop/services/prompts.py +76 -0
- strop/services/report.py +194 -0
- strop/services/review.py +43 -0
- strop/services/scaffold.py +79 -0
- strop/services/turn.py +234 -0
- strop/storage/__init__.py +6 -0
- strop/storage/cache.py +45 -0
- strop/storage/database.py +45 -0
- strop/storage/datasets.py +129 -0
- strop/storage/evaluations.py +160 -0
- strop/storage/migrations/001_initial.sql +5 -0
- strop/storage/migrations/002_datasets.sql +27 -0
- strop/storage/migrations/003_prompt_versions.sql +22 -0
- strop/storage/migrations/004_llm_cache.sql +6 -0
- strop/storage/migrations/005_evaluations.sql +37 -0
- strop/storage/migrations/006_runs.sql +52 -0
- strop/storage/migrations/007_example_feedback.sql +2 -0
- strop/storage/migrations/__init__.py +5 -0
- strop/storage/migrator.py +91 -0
- strop/storage/paths.py +30 -0
- strop/storage/prompts.py +96 -0
- strop/storage/run_models.py +75 -0
- strop/storage/runs.py +263 -0
- strop_cli-0.1.0.dist-info/METADATA +242 -0
- strop_cli-0.1.0.dist-info/RECORD +137 -0
- strop_cli-0.1.0.dist-info/WHEEL +4 -0
- strop_cli-0.1.0.dist-info/entry_points.txt +2 -0
- strop_cli-0.1.0.dist-info/licenses/LICENSE +21 -0
strop/errors.py
ADDED
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
"""Exception hierarchy and process exit codes.
|
|
2
|
+
|
|
3
|
+
The command-line layer is the only place that turns an exception into an exit
|
|
4
|
+
code. Library code raises a specific ``StropError`` subclass and stays unaware
|
|
5
|
+
of how the process ends.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from enum import IntEnum
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
class ExitCode(IntEnum):
|
|
12
|
+
"""Process exit codes documented in the CLI help."""
|
|
13
|
+
|
|
14
|
+
SUCCESS = 0
|
|
15
|
+
FAILURE = 1
|
|
16
|
+
USAGE = 2
|
|
17
|
+
PAUSED = 10
|
|
18
|
+
BUDGET = 11
|
|
19
|
+
INTERRUPTED = 12
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class StropError(Exception):
|
|
23
|
+
"""Base class for all expected, user-facing errors."""
|
|
24
|
+
|
|
25
|
+
exit_code: ExitCode = ExitCode.FAILURE
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
class ConfigError(StropError):
|
|
29
|
+
"""The project configuration or a command-line value is missing or invalid."""
|
|
30
|
+
|
|
31
|
+
exit_code = ExitCode.USAGE
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
class DataError(StropError):
|
|
35
|
+
"""A dataset file cannot be read or fails validation."""
|
|
36
|
+
|
|
37
|
+
exit_code = ExitCode.USAGE
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
class StorageError(StropError):
|
|
41
|
+
"""The local project database or its files cannot be used."""
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
class LLMError(StropError):
|
|
45
|
+
"""A model provider call failed after retries."""
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
class TransientLLMError(LLMError):
|
|
49
|
+
"""A temporary provider failure (rate limit, timeout, outage) that may succeed on retry."""
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
class InvalidModelReplyError(LLMError):
|
|
53
|
+
"""The model kept returning replies that do not match the requested JSON schema."""
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
class BudgetExceededError(StropError):
|
|
57
|
+
"""A configured call, cost, or time budget was reached."""
|
|
58
|
+
|
|
59
|
+
exit_code = ExitCode.BUDGET
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""Running prompts over data and scoring the outputs."""
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
"""Tracking calls, cost and time against the configured budgets."""
|
|
2
|
+
|
|
3
|
+
import time
|
|
4
|
+
from collections.abc import Callable
|
|
5
|
+
from dataclasses import dataclass, field
|
|
6
|
+
|
|
7
|
+
from strop.config.limits import BudgetsConfig
|
|
8
|
+
from strop.errors import BudgetExceededError
|
|
9
|
+
from strop.llm.types import LLMResponse
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
@dataclass
|
|
13
|
+
class BudgetTracker:
|
|
14
|
+
"""Counts fresh model calls and known cost, and raises once a limit is reached.
|
|
15
|
+
|
|
16
|
+
Cached responses are free and are not counted. A cost limit can only be enforced
|
|
17
|
+
for calls whose provider reported a cost; ``cost_unknown_calls`` shows the rest.
|
|
18
|
+
"""
|
|
19
|
+
|
|
20
|
+
config: BudgetsConfig
|
|
21
|
+
clock: Callable[[], float] = time.monotonic
|
|
22
|
+
calls: int = 0
|
|
23
|
+
cost_usd: float = 0.0
|
|
24
|
+
cost_unknown_calls: int = 0
|
|
25
|
+
_start: float = field(init=False)
|
|
26
|
+
|
|
27
|
+
def __post_init__(self) -> None:
|
|
28
|
+
self._start = self.clock()
|
|
29
|
+
|
|
30
|
+
@property
|
|
31
|
+
def elapsed_minutes(self) -> float:
|
|
32
|
+
return (self.clock() - self._start) / 60
|
|
33
|
+
|
|
34
|
+
def record(self, response: LLMResponse) -> None:
|
|
35
|
+
if response.cached:
|
|
36
|
+
return
|
|
37
|
+
self.calls += 1
|
|
38
|
+
if response.usage.cost_usd is None:
|
|
39
|
+
self.cost_unknown_calls += 1
|
|
40
|
+
else:
|
|
41
|
+
self.cost_usd += response.usage.cost_usd
|
|
42
|
+
|
|
43
|
+
def record_failed_call(self) -> None:
|
|
44
|
+
self.calls += 1
|
|
45
|
+
self.cost_unknown_calls += 1
|
|
46
|
+
|
|
47
|
+
def check(self) -> None:
|
|
48
|
+
"""Raise ``BudgetExceededError`` if any configured limit has been reached."""
|
|
49
|
+
limits = self.config
|
|
50
|
+
if limits.max_total_model_calls is not None and self.calls >= limits.max_total_model_calls:
|
|
51
|
+
raise BudgetExceededError(f"model call budget reached ({self.calls} calls)")
|
|
52
|
+
if limits.max_total_cost_usd is not None and self.cost_usd >= limits.max_total_cost_usd:
|
|
53
|
+
raise BudgetExceededError(f"cost budget reached (${self.cost_usd:.4f})")
|
|
54
|
+
if (
|
|
55
|
+
limits.max_wall_time_minutes is not None
|
|
56
|
+
and self.elapsed_minutes >= limits.max_wall_time_minutes
|
|
57
|
+
):
|
|
58
|
+
raise BudgetExceededError(f"time budget reached ({self.elapsed_minutes:.1f} minutes)")
|
|
59
|
+
|
|
60
|
+
def warnings(self) -> list[str]:
|
|
61
|
+
"""Caveats about limits that cannot be fully enforced."""
|
|
62
|
+
if self.config.max_total_cost_usd is not None and self.cost_unknown_calls:
|
|
63
|
+
return [
|
|
64
|
+
f"{self.cost_unknown_calls} call(s) reported no cost, so the cost budget "
|
|
65
|
+
"only covers the calls that did"
|
|
66
|
+
]
|
|
67
|
+
return []
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
"""A client wrapper that counts every call against the run budget."""
|
|
2
|
+
|
|
3
|
+
from strop.evaluation.budget import BudgetTracker
|
|
4
|
+
from strop.llm.client import LLMClient
|
|
5
|
+
from strop.llm.types import LLMRequest, LLMResponse
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class BudgetedClient:
|
|
9
|
+
"""Checks the budget before each call and records the call afterwards.
|
|
10
|
+
|
|
11
|
+
Used for the optimizer, analyzer and judge. Target-model calls are counted by
|
|
12
|
+
``PromptRunner`` instead, so they must not go through this wrapper as well.
|
|
13
|
+
"""
|
|
14
|
+
|
|
15
|
+
def __init__(self, inner: LLMClient, budget: BudgetTracker) -> None:
|
|
16
|
+
self._inner = inner
|
|
17
|
+
self._budget = budget
|
|
18
|
+
|
|
19
|
+
async def complete(self, request: LLMRequest) -> LLMResponse:
|
|
20
|
+
self._budget.check()
|
|
21
|
+
response = await self._inner.complete(request)
|
|
22
|
+
self._budget.record(response)
|
|
23
|
+
return response
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
"""Joining stored outcomes with their dataset rows to form scorable examples."""
|
|
2
|
+
|
|
3
|
+
from collections.abc import Mapping, Sequence
|
|
4
|
+
from dataclasses import replace
|
|
5
|
+
|
|
6
|
+
from strop.data.models import Row
|
|
7
|
+
from strop.evaluation.models import ExampleOutcome
|
|
8
|
+
from strop.metrics.suite import MetricSuite
|
|
9
|
+
from strop.metrics.types import Example
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def to_examples(rows: Sequence[Row], outcomes: Sequence[ExampleOutcome]) -> list[Example]:
|
|
13
|
+
"""Pair each outcome with the inputs of its row."""
|
|
14
|
+
by_id: Mapping[str, Row] = {row.id: row for row in rows}
|
|
15
|
+
return [
|
|
16
|
+
Example(
|
|
17
|
+
row_id=o.row_id,
|
|
18
|
+
inputs=by_id[o.row_id].inputs,
|
|
19
|
+
expected=o.expected,
|
|
20
|
+
raw_output=o.raw_output,
|
|
21
|
+
value=o.value,
|
|
22
|
+
parsed=o.parsed,
|
|
23
|
+
error=o.error,
|
|
24
|
+
)
|
|
25
|
+
for o in outcomes
|
|
26
|
+
]
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def with_scores(
|
|
30
|
+
outcomes: Sequence[ExampleOutcome], examples: Sequence[Example], suite: MetricSuite
|
|
31
|
+
) -> list[ExampleOutcome]:
|
|
32
|
+
"""Attach each example's per-metric scores and metric feedback to its outcome."""
|
|
33
|
+
return [
|
|
34
|
+
replace(
|
|
35
|
+
outcome,
|
|
36
|
+
scores={name: suite.metric(name).per_example(example) for name in suite.names},
|
|
37
|
+
feedback=_feedback(example, suite),
|
|
38
|
+
)
|
|
39
|
+
for outcome, example in zip(outcomes, examples, strict=True)
|
|
40
|
+
]
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def _feedback(example: Example, suite: MetricSuite) -> str | None:
|
|
44
|
+
notes = []
|
|
45
|
+
for name in suite.names:
|
|
46
|
+
text = suite.metric(name).feedback(example)
|
|
47
|
+
if text:
|
|
48
|
+
notes.append(f"{name}: {text}")
|
|
49
|
+
return "; ".join(notes) or None
|
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
"""Evaluation result types."""
|
|
2
|
+
|
|
3
|
+
from collections.abc import Sequence
|
|
4
|
+
from dataclasses import dataclass, field
|
|
5
|
+
from typing import Any
|
|
6
|
+
|
|
7
|
+
from strop.data.models import Partition
|
|
8
|
+
from strop.metrics.suite import MetricReport
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
@dataclass(frozen=True, slots=True)
|
|
12
|
+
class ExampleOutcome:
|
|
13
|
+
"""The stored result for one row: the model output, its parse, and its per-metric scores."""
|
|
14
|
+
|
|
15
|
+
row_id: str
|
|
16
|
+
expected: str | None
|
|
17
|
+
raw_output: str
|
|
18
|
+
value: str | None
|
|
19
|
+
parsed: Any = None
|
|
20
|
+
error: str | None = None
|
|
21
|
+
finish_reason: str | None = None
|
|
22
|
+
latency_s: float = 0.0
|
|
23
|
+
prompt_tokens: int | None = None
|
|
24
|
+
completion_tokens: int | None = None
|
|
25
|
+
total_tokens: int | None = None
|
|
26
|
+
cost_usd: float | None = None
|
|
27
|
+
cached: bool = False
|
|
28
|
+
detail: str | None = None
|
|
29
|
+
feedback: str | None = None
|
|
30
|
+
scores: dict[str, float | None] = field(default_factory=dict)
|
|
31
|
+
|
|
32
|
+
@property
|
|
33
|
+
def failed(self) -> bool:
|
|
34
|
+
return self.error is not None
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
@dataclass(frozen=True, slots=True)
|
|
38
|
+
class UsageSummary:
|
|
39
|
+
"""Calls, tokens, cost and latency. ``None`` means the provider reported nothing."""
|
|
40
|
+
|
|
41
|
+
calls: int = 0
|
|
42
|
+
cached_calls: int = 0
|
|
43
|
+
prompt_tokens: int | None = None
|
|
44
|
+
completion_tokens: int | None = None
|
|
45
|
+
total_tokens: int | None = None
|
|
46
|
+
cost_usd: float | None = None
|
|
47
|
+
cost_known_calls: int = 0
|
|
48
|
+
mean_latency_s: float | None = None
|
|
49
|
+
|
|
50
|
+
@property
|
|
51
|
+
def cost_complete(self) -> bool:
|
|
52
|
+
"""True if every non-cached call reported a cost."""
|
|
53
|
+
return self.cost_known_calls == self.calls - self.cached_calls
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def _sum_known(values: Sequence[int | float | None]) -> Any:
|
|
57
|
+
known = [v for v in values if v is not None]
|
|
58
|
+
return sum(known) if known else None
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def summarize_usage(outcomes: Sequence[ExampleOutcome]) -> UsageSummary:
|
|
62
|
+
"""Totals over ``outcomes``. Cached answers add no tokens or cost and no new call."""
|
|
63
|
+
fresh = [o for o in outcomes if not o.cached]
|
|
64
|
+
latencies = [o.latency_s for o in fresh if o.latency_s > 0]
|
|
65
|
+
return UsageSummary(
|
|
66
|
+
calls=len(outcomes),
|
|
67
|
+
cached_calls=len(outcomes) - len(fresh),
|
|
68
|
+
prompt_tokens=_sum_known([o.prompt_tokens for o in fresh]),
|
|
69
|
+
completion_tokens=_sum_known([o.completion_tokens for o in fresh]),
|
|
70
|
+
total_tokens=_sum_known([o.total_tokens for o in fresh]),
|
|
71
|
+
cost_usd=_sum_known([o.cost_usd for o in fresh]),
|
|
72
|
+
cost_known_calls=sum(o.cost_usd is not None for o in fresh),
|
|
73
|
+
mean_latency_s=sum(latencies) / len(latencies) if latencies else None,
|
|
74
|
+
)
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
@dataclass(frozen=True)
|
|
78
|
+
class EvaluationRecord:
|
|
79
|
+
"""A stored evaluation of one prompt version on one partition."""
|
|
80
|
+
|
|
81
|
+
id: int
|
|
82
|
+
prompt_number: int
|
|
83
|
+
split_id: int
|
|
84
|
+
partition: Partition
|
|
85
|
+
kind: str
|
|
86
|
+
report: MetricReport
|
|
87
|
+
usage: UsageSummary
|
|
88
|
+
sample_count: int
|
|
89
|
+
error_count: int
|
|
90
|
+
created_at: str
|
|
91
|
+
details: dict[str, Any] = field(default_factory=dict)
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
@dataclass(frozen=True)
|
|
95
|
+
class NewEvaluation:
|
|
96
|
+
"""Everything needed to store an evaluation."""
|
|
97
|
+
|
|
98
|
+
prompt_number: int
|
|
99
|
+
split_id: int
|
|
100
|
+
partition: Partition
|
|
101
|
+
kind: str
|
|
102
|
+
report: MetricReport
|
|
103
|
+
outcomes: Sequence[ExampleOutcome]
|
|
104
|
+
details: dict[str, Any] = field(default_factory=dict)
|
|
105
|
+
run_id: int | None = None
|
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
"""Turning raw model text into a value that metrics can score."""
|
|
2
|
+
|
|
3
|
+
import json
|
|
4
|
+
import re
|
|
5
|
+
from dataclasses import dataclass
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
from typing import Any
|
|
8
|
+
|
|
9
|
+
from jsonschema import Draft202012Validator
|
|
10
|
+
from jsonschema.exceptions import SchemaError
|
|
11
|
+
|
|
12
|
+
from strop.config.enums import OutputType
|
|
13
|
+
from strop.config.output import OutputConfig
|
|
14
|
+
from strop.errors import ConfigError
|
|
15
|
+
from strop.metrics.types import ErrorCode
|
|
16
|
+
|
|
17
|
+
_FENCE = re.compile(r"^```(?:json)?\s*(.*?)\s*```$", re.DOTALL | re.IGNORECASE)
|
|
18
|
+
_QUOTES = "`'\"" + "".join(chr(code) for code in (0x201C, 0x201D, 0x2018, 0x2019))
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
@dataclass(frozen=True, slots=True)
|
|
22
|
+
class ParsedOutput:
|
|
23
|
+
value: str | None
|
|
24
|
+
parsed: Any = None
|
|
25
|
+
error: ErrorCode | None = None
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def canonical_json(value: Any) -> str:
|
|
29
|
+
return json.dumps(value, sort_keys=True, ensure_ascii=False)
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def canonical_expected(text: str, output_type: OutputType) -> str:
|
|
33
|
+
"""Normalize an expected value the same way outputs are normalized."""
|
|
34
|
+
stripped = text.strip()
|
|
35
|
+
if output_type is OutputType.JSON:
|
|
36
|
+
return canonical_json(json.loads(stripped))
|
|
37
|
+
return stripped
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def load_output_schema(root: Path, output: OutputConfig) -> dict[str, Any] | None:
|
|
41
|
+
"""Read and check the JSON schema named in the output contract, if any."""
|
|
42
|
+
if output.schema_file is None:
|
|
43
|
+
return None
|
|
44
|
+
path = root / output.schema_file
|
|
45
|
+
try:
|
|
46
|
+
schema = json.loads(path.read_text(encoding="utf-8"))
|
|
47
|
+
Draft202012Validator.check_schema(schema)
|
|
48
|
+
except FileNotFoundError:
|
|
49
|
+
raise ConfigError(f"output schema file not found: {path}") from None
|
|
50
|
+
except (ValueError, SchemaError) as exc:
|
|
51
|
+
raise ConfigError(f"output schema {path.name} is not a valid JSON schema: {exc}") from exc
|
|
52
|
+
return schema # type: ignore[no-any-return]
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def parse_output(
|
|
56
|
+
raw: str, finish_reason: str | None, output: OutputConfig, schema: dict[str, Any] | None = None
|
|
57
|
+
) -> ParsedOutput:
|
|
58
|
+
"""Normalize ``raw`` under the output contract; failures carry an error code."""
|
|
59
|
+
text = raw.strip()
|
|
60
|
+
if not text:
|
|
61
|
+
return ParsedOutput(None, error=ErrorCode.EMPTY_OUTPUT)
|
|
62
|
+
failure = ErrorCode.TRUNCATED if finish_reason == "length" else ErrorCode.INVALID_FORMAT
|
|
63
|
+
if output.type is OutputType.LABEL:
|
|
64
|
+
label = _match_label(text, output)
|
|
65
|
+
return ParsedOutput(label) if label else ParsedOutput(None, error=failure)
|
|
66
|
+
if output.type is OutputType.JSON:
|
|
67
|
+
return _parse_json(text, schema, failure)
|
|
68
|
+
if finish_reason == "length":
|
|
69
|
+
return ParsedOutput(None, error=ErrorCode.TRUNCATED)
|
|
70
|
+
return ParsedOutput(text)
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def _match_label(text: str, output: OutputConfig) -> str | None:
|
|
74
|
+
by_name = {label.casefold(): label for label in output.labels}
|
|
75
|
+
candidate = text.strip(_QUOTES + " \t\r\n").rstrip(".!").strip(_QUOTES + " ")
|
|
76
|
+
if candidate.casefold() in by_name:
|
|
77
|
+
return by_name[candidate.casefold()]
|
|
78
|
+
if output.match != "contains":
|
|
79
|
+
return None
|
|
80
|
+
found = [
|
|
81
|
+
label
|
|
82
|
+
for key, label in by_name.items()
|
|
83
|
+
if re.search(rf"(?<!\w){re.escape(key)}(?!\w)", text.casefold())
|
|
84
|
+
]
|
|
85
|
+
return found[0] if len(found) == 1 else None
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def _parse_json(text: str, schema: dict[str, Any] | None, failure: ErrorCode) -> ParsedOutput:
|
|
89
|
+
fenced = _FENCE.match(text)
|
|
90
|
+
try:
|
|
91
|
+
parsed = json.loads(fenced.group(1) if fenced else text)
|
|
92
|
+
except ValueError:
|
|
93
|
+
return ParsedOutput(None, error=failure)
|
|
94
|
+
if schema is not None and next(Draft202012Validator(schema).iter_errors(parsed), None):
|
|
95
|
+
return ParsedOutput(None, parsed, ErrorCode.SCHEMA_VIOLATION)
|
|
96
|
+
return ParsedOutput(canonical_json(parsed), parsed)
|
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
"""Running a prompt over rows with bounded concurrency."""
|
|
2
|
+
|
|
3
|
+
import asyncio
|
|
4
|
+
from collections.abc import Callable, Sequence
|
|
5
|
+
from typing import Any
|
|
6
|
+
|
|
7
|
+
from strop.config.limits import ExecutionConfig
|
|
8
|
+
from strop.config.llm import ModelSpec
|
|
9
|
+
from strop.config.output import OutputConfig
|
|
10
|
+
from strop.data.models import Row
|
|
11
|
+
from strop.errors import LLMError
|
|
12
|
+
from strop.evaluation.budget import BudgetTracker
|
|
13
|
+
from strop.evaluation.models import ExampleOutcome
|
|
14
|
+
from strop.evaluation.parsing import canonical_expected, parse_output
|
|
15
|
+
from strop.llm.client import LLMClient
|
|
16
|
+
from strop.llm.types import LLMRequest, LLMResponse
|
|
17
|
+
from strop.log import redact
|
|
18
|
+
from strop.metrics.types import ErrorCode
|
|
19
|
+
from strop.prompts import PromptTemplate, render_messages
|
|
20
|
+
|
|
21
|
+
Progress = Callable[[int, int], None]
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
class PromptRunner:
|
|
25
|
+
"""Sends one request per row and turns each reply into an ``ExampleOutcome``."""
|
|
26
|
+
|
|
27
|
+
def __init__(
|
|
28
|
+
self,
|
|
29
|
+
client: LLMClient,
|
|
30
|
+
spec: ModelSpec,
|
|
31
|
+
output: OutputConfig,
|
|
32
|
+
schema: dict[str, Any] | None,
|
|
33
|
+
execution: ExecutionConfig,
|
|
34
|
+
budget: BudgetTracker | None = None,
|
|
35
|
+
) -> None:
|
|
36
|
+
self._client = client
|
|
37
|
+
self._spec = spec
|
|
38
|
+
self._output = output
|
|
39
|
+
self._schema = schema
|
|
40
|
+
self._execution = execution
|
|
41
|
+
self._budget = budget
|
|
42
|
+
|
|
43
|
+
async def run(
|
|
44
|
+
self,
|
|
45
|
+
template: PromptTemplate,
|
|
46
|
+
rows: Sequence[Row],
|
|
47
|
+
progress: Progress | None = None,
|
|
48
|
+
) -> list[ExampleOutcome]:
|
|
49
|
+
"""Evaluate ``rows`` in batches, checking the budget between batches."""
|
|
50
|
+
semaphore = asyncio.Semaphore(self._execution.max_concurrency)
|
|
51
|
+
outcomes: list[ExampleOutcome] = []
|
|
52
|
+
for start in range(0, len(rows), self._execution.batch_size):
|
|
53
|
+
if self._budget is not None:
|
|
54
|
+
self._budget.check()
|
|
55
|
+
batch = rows[start : start + self._execution.batch_size]
|
|
56
|
+
outcomes.extend(
|
|
57
|
+
await asyncio.gather(*(self._one(template, row, semaphore) for row in batch))
|
|
58
|
+
)
|
|
59
|
+
if progress is not None:
|
|
60
|
+
progress(len(outcomes), len(rows))
|
|
61
|
+
return outcomes
|
|
62
|
+
|
|
63
|
+
async def _one(
|
|
64
|
+
self, template: PromptTemplate, row: Row, semaphore: asyncio.Semaphore
|
|
65
|
+
) -> ExampleOutcome:
|
|
66
|
+
expected = (
|
|
67
|
+
canonical_expected(row.expected, self._output.type)
|
|
68
|
+
if row.expected is not None
|
|
69
|
+
else None
|
|
70
|
+
)
|
|
71
|
+
request = LLMRequest.for_model(self._spec, render_messages(template, row.inputs))
|
|
72
|
+
async with semaphore:
|
|
73
|
+
try:
|
|
74
|
+
response = await self._client.complete(request)
|
|
75
|
+
except LLMError as exc:
|
|
76
|
+
if self._budget is not None:
|
|
77
|
+
self._budget.record_failed_call()
|
|
78
|
+
return ExampleOutcome(
|
|
79
|
+
row.id,
|
|
80
|
+
expected,
|
|
81
|
+
"",
|
|
82
|
+
None,
|
|
83
|
+
error=ErrorCode.PROVIDER_ERROR,
|
|
84
|
+
detail=redact(str(exc)),
|
|
85
|
+
)
|
|
86
|
+
if self._budget is not None:
|
|
87
|
+
self._budget.record(response)
|
|
88
|
+
return self._outcome(row, expected, response)
|
|
89
|
+
|
|
90
|
+
def _outcome(self, row: Row, expected: str | None, response: LLMResponse) -> ExampleOutcome:
|
|
91
|
+
parsed = parse_output(response.text, response.finish_reason, self._output, self._schema)
|
|
92
|
+
usage = response.usage
|
|
93
|
+
return ExampleOutcome(
|
|
94
|
+
row_id=row.id,
|
|
95
|
+
expected=expected,
|
|
96
|
+
raw_output=response.text,
|
|
97
|
+
value=parsed.value,
|
|
98
|
+
parsed=parsed.parsed,
|
|
99
|
+
error=parsed.error,
|
|
100
|
+
finish_reason=response.finish_reason,
|
|
101
|
+
latency_s=response.latency_s,
|
|
102
|
+
prompt_tokens=usage.prompt_tokens,
|
|
103
|
+
completion_tokens=usage.completion_tokens,
|
|
104
|
+
total_tokens=usage.total_tokens,
|
|
105
|
+
cost_usd=usage.cost_usd,
|
|
106
|
+
cached=response.cached,
|
|
107
|
+
)
|
strop/formatting.py
ADDED
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
"""Small text helpers shared by services and the terminal views."""
|
|
2
|
+
|
|
3
|
+
MAX_TEXT = 70
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
def fmt(value: float | None, digits: int = 4) -> str:
|
|
7
|
+
"""A number with fixed decimals, or ``n/a`` when it is unavailable."""
|
|
8
|
+
return "n/a" if value is None else f"{value:.{digits}f}"
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def shorten(text: str, limit: int = MAX_TEXT) -> str:
|
|
12
|
+
"""Collapse whitespace and cut ``text`` to ``limit`` characters."""
|
|
13
|
+
flat = " ".join(text.split())
|
|
14
|
+
return flat if len(flat) <= limit else flat[: limit - 3] + "..."
|
strop/llm/__init__.py
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
"""Model access behind a provider-independent interface."""
|
|
2
|
+
|
|
3
|
+
from strop.llm.client import LLMClient
|
|
4
|
+
from strop.llm.mock import MockLLMClient
|
|
5
|
+
from strop.llm.types import LLMRequest, LLMResponse, Message, Usage
|
|
6
|
+
|
|
7
|
+
__all__ = ["LLMClient", "LLMRequest", "LLMResponse", "Message", "MockLLMClient", "Usage"]
|
strop/llm/caching.py
ADDED
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
"""Response caching around any client."""
|
|
2
|
+
|
|
3
|
+
from strop.llm.client import LLMClient
|
|
4
|
+
from strop.llm.types import LLMRequest, LLMResponse
|
|
5
|
+
from strop.storage.cache import ResponseCache, cache_key
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class CachingClient:
|
|
9
|
+
"""Serves repeated deterministic requests from the cache.
|
|
10
|
+
|
|
11
|
+
Only requests at temperature 0 are cached: sampling is meant to vary between calls,
|
|
12
|
+
so a cached answer would silently remove that variety.
|
|
13
|
+
"""
|
|
14
|
+
|
|
15
|
+
def __init__(self, inner: LLMClient, cache: ResponseCache) -> None:
|
|
16
|
+
self._inner = inner
|
|
17
|
+
self._cache = cache
|
|
18
|
+
|
|
19
|
+
async def complete(self, request: LLMRequest) -> LLMResponse:
|
|
20
|
+
if request.temperature > 0:
|
|
21
|
+
return await self._inner.complete(request)
|
|
22
|
+
key = cache_key(request)
|
|
23
|
+
hit = self._cache.get(key)
|
|
24
|
+
if hit is not None:
|
|
25
|
+
return hit
|
|
26
|
+
response = await self._inner.complete(request)
|
|
27
|
+
self._cache.put(key, response)
|
|
28
|
+
return response
|
strop/llm/client.py
ADDED
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
"""The interface every model provider implementation satisfies."""
|
|
2
|
+
|
|
3
|
+
from typing import Protocol, runtime_checkable
|
|
4
|
+
|
|
5
|
+
from strop.llm.types import LLMRequest, LLMResponse
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
@runtime_checkable
|
|
9
|
+
class LLMClient(Protocol):
|
|
10
|
+
"""Sends one chat request and returns a normalized response.
|
|
11
|
+
|
|
12
|
+
Implementations raise ``strop.errors.LLMError`` when a call fails for good.
|
|
13
|
+
"""
|
|
14
|
+
|
|
15
|
+
async def complete(self, request: LLMRequest) -> LLMResponse: ...
|
strop/llm/factory.py
ADDED
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
"""Assembling the production client stack."""
|
|
2
|
+
|
|
3
|
+
from strop.config.limits import ExecutionConfig
|
|
4
|
+
from strop.llm.caching import CachingClient
|
|
5
|
+
from strop.llm.client import LLMClient
|
|
6
|
+
from strop.llm.litellm_client import LiteLLMClient
|
|
7
|
+
from strop.llm.retry import RetryingClient
|
|
8
|
+
from strop.storage.cache import ResponseCache
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def build_client(execution: ExecutionConfig, cache: ResponseCache | None = None) -> LLMClient:
|
|
12
|
+
"""LiteLLM with retries, and a response cache when ``cache`` is given."""
|
|
13
|
+
client: LLMClient = RetryingClient(LiteLLMClient(), attempts=execution.retry_attempts)
|
|
14
|
+
return CachingClient(client, cache) if cache is not None else client
|
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
"""The LiteLLM-backed client. LiteLLM is imported lazily to keep CLI start-up fast."""
|
|
2
|
+
|
|
3
|
+
import logging
|
|
4
|
+
import os
|
|
5
|
+
import time
|
|
6
|
+
from types import ModuleType
|
|
7
|
+
from typing import Any
|
|
8
|
+
|
|
9
|
+
from strop.errors import LLMError, TransientLLMError
|
|
10
|
+
from strop.llm.types import LLMRequest, LLMResponse, Usage
|
|
11
|
+
from strop.log import redact
|
|
12
|
+
|
|
13
|
+
_TRANSIENT_NAMES = (
|
|
14
|
+
"RateLimitError",
|
|
15
|
+
"Timeout",
|
|
16
|
+
"APIConnectionError",
|
|
17
|
+
"ServiceUnavailableError",
|
|
18
|
+
"InternalServerError",
|
|
19
|
+
)
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def _load_litellm() -> ModuleType:
|
|
23
|
+
# Use the price table bundled with LiteLLM instead of downloading one when it is imported.
|
|
24
|
+
os.environ.setdefault("LITELLM_LOCAL_MODEL_COST_MAP", "True")
|
|
25
|
+
import litellm # noqa: PLC0415 - deferred: importing litellm takes over a second
|
|
26
|
+
|
|
27
|
+
litellm.suppress_debug_info = True
|
|
28
|
+
litellm.telemetry = False
|
|
29
|
+
logging.getLogger("LiteLLM").setLevel(logging.ERROR)
|
|
30
|
+
return litellm
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
class LiteLLMClient:
|
|
34
|
+
"""Calls any provider and model LiteLLM supports and normalizes the response."""
|
|
35
|
+
|
|
36
|
+
async def complete(self, request: LLMRequest) -> LLMResponse:
|
|
37
|
+
litellm = _load_litellm()
|
|
38
|
+
started = time.perf_counter()
|
|
39
|
+
try:
|
|
40
|
+
raw = await litellm.acompletion(
|
|
41
|
+
model=request.model,
|
|
42
|
+
messages=[{"role": m.role, "content": m.content} for m in request.messages],
|
|
43
|
+
temperature=request.temperature,
|
|
44
|
+
max_tokens=request.max_tokens,
|
|
45
|
+
timeout=request.timeout_s,
|
|
46
|
+
drop_params=True,
|
|
47
|
+
**request.params,
|
|
48
|
+
)
|
|
49
|
+
except Exception as exc:
|
|
50
|
+
raise _translate(litellm, exc) from exc
|
|
51
|
+
return _normalize(litellm, raw, request, time.perf_counter() - started)
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def _translate(litellm: ModuleType, exc: Exception) -> LLMError:
|
|
55
|
+
transient = tuple(
|
|
56
|
+
cls for name in _TRANSIENT_NAMES if isinstance(cls := getattr(litellm, name, None), type)
|
|
57
|
+
)
|
|
58
|
+
message = redact(f"{type(exc).__name__}: {exc}")
|
|
59
|
+
return TransientLLMError(message) if isinstance(exc, transient) else LLMError(message)
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def _normalize(litellm: ModuleType, raw: Any, request: LLMRequest, latency_s: float) -> LLMResponse:
|
|
63
|
+
try:
|
|
64
|
+
choice = raw.choices[0]
|
|
65
|
+
text = choice.message.content or ""
|
|
66
|
+
finish_reason = choice.finish_reason
|
|
67
|
+
except (AttributeError, IndexError, TypeError) as exc:
|
|
68
|
+
raise LLMError("the provider returned a response without a message") from exc
|
|
69
|
+
return LLMResponse(
|
|
70
|
+
text=text,
|
|
71
|
+
model=getattr(raw, "model", None) or request.model,
|
|
72
|
+
finish_reason=finish_reason,
|
|
73
|
+
usage=_usage(litellm, raw),
|
|
74
|
+
latency_s=latency_s,
|
|
75
|
+
request_id=getattr(raw, "id", None),
|
|
76
|
+
)
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def _usage(litellm: ModuleType, raw: Any) -> Usage:
|
|
80
|
+
reported = getattr(raw, "usage", None)
|
|
81
|
+
try:
|
|
82
|
+
cost = float(litellm.completion_cost(completion_response=raw))
|
|
83
|
+
except Exception:
|
|
84
|
+
cost = None
|
|
85
|
+
return Usage(
|
|
86
|
+
prompt_tokens=getattr(reported, "prompt_tokens", None),
|
|
87
|
+
completion_tokens=getattr(reported, "completion_tokens", None),
|
|
88
|
+
total_tokens=getattr(reported, "total_tokens", None),
|
|
89
|
+
cost_usd=cost,
|
|
90
|
+
)
|