strop-cli 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- strop/__init__.py +5 -0
- strop/__main__.py +5 -0
- strop/analysis/__init__.py +17 -0
- strop/analysis/classify.py +52 -0
- strop/analysis/evidence.py +52 -0
- strop/analysis/facts.py +74 -0
- strop/analysis/patterns.py +90 -0
- strop/cli/__init__.py +5 -0
- strop/cli/analyze.py +93 -0
- strop/cli/app.py +82 -0
- strop/cli/data.py +111 -0
- strop/cli/evaluate.py +145 -0
- strop/cli/export.py +74 -0
- strop/cli/final.py +106 -0
- strop/cli/init.py +131 -0
- strop/cli/optimize.py +156 -0
- strop/cli/options.py +20 -0
- strop/cli/progress.py +18 -0
- strop/cli/prompts.py +110 -0
- strop/cli/status.py +68 -0
- strop/cli/versions.py +123 -0
- strop/config/__init__.py +16 -0
- strop/config/base.py +9 -0
- strop/config/constraints.py +33 -0
- strop/config/dataset.py +63 -0
- strop/config/enums.py +52 -0
- strop/config/io.py +52 -0
- strop/config/limits.py +19 -0
- strop/config/llm.py +42 -0
- strop/config/metrics.py +86 -0
- strop/config/optimization.py +23 -0
- strop/config/output.py +31 -0
- strop/config/overrides.py +22 -0
- strop/config/root.py +36 -0
- strop/config/rules.py +67 -0
- strop/config/split.py +23 -0
- strop/data/__init__.py +58 -0
- strop/data/loaders.py +85 -0
- strop/data/models.py +44 -0
- strop/data/rows.py +99 -0
- strop/data/splitting.py +103 -0
- strop/env.py +44 -0
- strop/errors.py +59 -0
- strop/evaluation/__init__.py +1 -0
- strop/evaluation/budget.py +67 -0
- strop/evaluation/budgeted.py +23 -0
- strop/evaluation/examples.py +49 -0
- strop/evaluation/models.py +105 -0
- strop/evaluation/parsing.py +96 -0
- strop/evaluation/runner.py +107 -0
- strop/formatting.py +14 -0
- strop/llm/__init__.py +7 -0
- strop/llm/caching.py +28 -0
- strop/llm/client.py +15 -0
- strop/llm/factory.py +14 -0
- strop/llm/litellm_client.py +90 -0
- strop/llm/mock.py +43 -0
- strop/llm/retry.py +44 -0
- strop/llm/structured.py +52 -0
- strop/llm/types.py +59 -0
- strop/log.py +49 -0
- strop/metrics/__init__.py +1 -0
- strop/metrics/basic.py +30 -0
- strop/metrics/classification.py +164 -0
- strop/metrics/judge.py +130 -0
- strop/metrics/python_metric.py +101 -0
- strop/metrics/python_runner.py +65 -0
- strop/metrics/registry.py +115 -0
- strop/metrics/rules.py +55 -0
- strop/metrics/structured.py +109 -0
- strop/metrics/suite.py +109 -0
- strop/metrics/text.py +55 -0
- strop/metrics/types.py +106 -0
- strop/optimization/__init__.py +25 -0
- strop/optimization/acceptance.py +170 -0
- strop/optimization/generator.py +92 -0
- strop/optimization/meta_prompt.py +101 -0
- strop/optimization/types.py +69 -0
- strop/prompts/__init__.py +18 -0
- strop/prompts/diff.py +139 -0
- strop/prompts/files.py +92 -0
- strop/prompts/models.py +51 -0
- strop/prompts/render.py +16 -0
- strop/prompts/template.py +45 -0
- strop/prompts/validation.py +31 -0
- strop/prompts/versions.py +34 -0
- strop/py.typed +0 -0
- strop/reporting/__init__.py +6 -0
- strop/reporting/console.py +20 -0
- strop/reporting/diff_view.py +102 -0
- strop/reporting/evaluation_view.py +141 -0
- strop/reporting/final_view.py +109 -0
- strop/reporting/prompt_export.py +58 -0
- strop/reporting/report_markdown.py +157 -0
- strop/reporting/run_view.py +118 -0
- strop/services/__init__.py +1 -0
- strop/services/analysis.py +124 -0
- strop/services/clean.py +38 -0
- strop/services/compare.py +60 -0
- strop/services/context.py +70 -0
- strop/services/data.py +65 -0
- strop/services/evaluate.py +105 -0
- strop/services/evaluator.py +169 -0
- strop/services/export.py +90 -0
- strop/services/final_test.py +69 -0
- strop/services/metrics.py +89 -0
- strop/services/optimize.py +187 -0
- strop/services/partitions.py +46 -0
- strop/services/project.py +55 -0
- strop/services/prompts.py +76 -0
- strop/services/report.py +194 -0
- strop/services/review.py +43 -0
- strop/services/scaffold.py +79 -0
- strop/services/turn.py +234 -0
- strop/storage/__init__.py +6 -0
- strop/storage/cache.py +45 -0
- strop/storage/database.py +45 -0
- strop/storage/datasets.py +129 -0
- strop/storage/evaluations.py +160 -0
- strop/storage/migrations/001_initial.sql +5 -0
- strop/storage/migrations/002_datasets.sql +27 -0
- strop/storage/migrations/003_prompt_versions.sql +22 -0
- strop/storage/migrations/004_llm_cache.sql +6 -0
- strop/storage/migrations/005_evaluations.sql +37 -0
- strop/storage/migrations/006_runs.sql +52 -0
- strop/storage/migrations/007_example_feedback.sql +2 -0
- strop/storage/migrations/__init__.py +5 -0
- strop/storage/migrator.py +91 -0
- strop/storage/paths.py +30 -0
- strop/storage/prompts.py +96 -0
- strop/storage/run_models.py +75 -0
- strop/storage/runs.py +263 -0
- strop_cli-0.1.0.dist-info/METADATA +242 -0
- strop_cli-0.1.0.dist-info/RECORD +137 -0
- strop_cli-0.1.0.dist-info/WHEEL +4 -0
- strop_cli-0.1.0.dist-info/entry_points.txt +2 -0
- strop_cli-0.1.0.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
"""The ``baseline`` and ``eval`` workflows."""
|
|
2
|
+
|
|
3
|
+
import random
|
|
4
|
+
import sqlite3
|
|
5
|
+
from collections.abc import Sequence
|
|
6
|
+
from dataclasses import dataclass, replace
|
|
7
|
+
|
|
8
|
+
from strop.data.models import Partition, Row
|
|
9
|
+
from strop.evaluation.models import EvaluationRecord
|
|
10
|
+
from strop.evaluation.runner import Progress
|
|
11
|
+
from strop.llm.factory import build_client
|
|
12
|
+
from strop.prompts.versions import PromptVersion
|
|
13
|
+
from strop.services.evaluator import RunTag, build_evaluator
|
|
14
|
+
from strop.services.project import Project
|
|
15
|
+
from strop.services.prompts import ensure_initial_version, resolve_version
|
|
16
|
+
from strop.storage.prompts import PromptRepository
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
@dataclass(frozen=True)
|
|
20
|
+
class BaselineOutcome:
|
|
21
|
+
version: PromptVersion
|
|
22
|
+
records: list[EvaluationRecord]
|
|
23
|
+
train: EvaluationRecord | None
|
|
24
|
+
noise: float | None
|
|
25
|
+
|
|
26
|
+
@property
|
|
27
|
+
def reference(self) -> EvaluationRecord:
|
|
28
|
+
"""The first validation run; later repeats only measure run-to-run noise."""
|
|
29
|
+
return self.records[0]
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def sample_rows(rows: Sequence[Row], size: int, seed: int) -> list[Row]:
|
|
33
|
+
"""A reproducible random subset of ``rows`` of at most ``size`` rows."""
|
|
34
|
+
if size >= len(rows):
|
|
35
|
+
return list(rows)
|
|
36
|
+
chosen = random.Random(seed).sample(sorted(rows, key=lambda r: r.id), size)
|
|
37
|
+
return sorted(chosen, key=lambda r: r.id)
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
async def run_baseline(
|
|
41
|
+
project: Project,
|
|
42
|
+
conn: sqlite3.Connection,
|
|
43
|
+
*,
|
|
44
|
+
repeats: int,
|
|
45
|
+
sample: int | None,
|
|
46
|
+
with_train: bool,
|
|
47
|
+
use_cache: bool,
|
|
48
|
+
progress: Progress | None = None,
|
|
49
|
+
) -> BaselineOutcome:
|
|
50
|
+
"""Evaluate the initial prompt (v1) on validation, optionally repeated and on train too.
|
|
51
|
+
|
|
52
|
+
Repeats after the first bypass the response cache, so they measure real run-to-run noise.
|
|
53
|
+
"""
|
|
54
|
+
version = ensure_initial_version(project, PromptRepository(conn))
|
|
55
|
+
evaluator = build_evaluator(project, conn, use_cache=use_cache)
|
|
56
|
+
rows = evaluator.rows(Partition.VALIDATION)
|
|
57
|
+
if sample is not None:
|
|
58
|
+
rows = sample_rows(rows, sample, evaluator.partitions.seed)
|
|
59
|
+
records = []
|
|
60
|
+
for repeat in range(1, repeats + 1):
|
|
61
|
+
runner = evaluator
|
|
62
|
+
if repeat > 1:
|
|
63
|
+
runner = replace(evaluator, client=build_client(project.config.execution, None))
|
|
64
|
+
details = {"repeat": repeat, "sampled": sample is not None}
|
|
65
|
+
tag = RunTag("baseline", details)
|
|
66
|
+
records.append(
|
|
67
|
+
await runner.evaluate(version, Partition.VALIDATION, tag, rows=rows, progress=progress)
|
|
68
|
+
)
|
|
69
|
+
train = None
|
|
70
|
+
if with_train:
|
|
71
|
+
train = await evaluator.evaluate(
|
|
72
|
+
version, Partition.TRAIN, RunTag("baseline"), progress=progress
|
|
73
|
+
)
|
|
74
|
+
return BaselineOutcome(version, records, train, _noise(records))
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
async def run_eval(
|
|
78
|
+
project: Project,
|
|
79
|
+
conn: sqlite3.Connection,
|
|
80
|
+
*,
|
|
81
|
+
ref: str | None,
|
|
82
|
+
partition: Partition,
|
|
83
|
+
sample: int | None,
|
|
84
|
+
use_cache: bool,
|
|
85
|
+
progress: Progress | None = None,
|
|
86
|
+
) -> tuple[PromptVersion, EvaluationRecord]:
|
|
87
|
+
"""Score one prompt version on a partition without changing anything."""
|
|
88
|
+
repo = PromptRepository(conn)
|
|
89
|
+
ensure_initial_version(project, repo)
|
|
90
|
+
version = resolve_version(repo, ref)
|
|
91
|
+
evaluator = build_evaluator(project, conn, use_cache=use_cache)
|
|
92
|
+
rows = evaluator.rows(partition)
|
|
93
|
+
if sample is not None:
|
|
94
|
+
rows = sample_rows(rows, sample, evaluator.partitions.seed)
|
|
95
|
+
tag = RunTag("eval", {"sampled": sample is not None})
|
|
96
|
+
record = await evaluator.evaluate(version, partition, tag, rows=rows, progress=progress)
|
|
97
|
+
return version, record
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def _noise(records: Sequence[EvaluationRecord]) -> float | None:
|
|
101
|
+
values = [r.report.primary_value.value for r in records]
|
|
102
|
+
known = [v for v in values if v is not None]
|
|
103
|
+
if len(records) < 2 or len(known) != len(values):
|
|
104
|
+
return None
|
|
105
|
+
return max(known) - min(known)
|
|
@@ -0,0 +1,169 @@
|
|
|
1
|
+
"""Evaluating prompt versions on dataset partitions and storing the results."""
|
|
2
|
+
|
|
3
|
+
import hashlib
|
|
4
|
+
import json
|
|
5
|
+
import logging
|
|
6
|
+
import sqlite3
|
|
7
|
+
from collections.abc import Sequence
|
|
8
|
+
from dataclasses import dataclass, field
|
|
9
|
+
from typing import Any
|
|
10
|
+
|
|
11
|
+
from strop.config.enums import ModelRole
|
|
12
|
+
from strop.config.llm import ModelSpec
|
|
13
|
+
from strop.data.models import Partition, Row
|
|
14
|
+
from strop.errors import DataError
|
|
15
|
+
from strop.evaluation.budget import BudgetTracker
|
|
16
|
+
from strop.evaluation.budgeted import BudgetedClient
|
|
17
|
+
from strop.evaluation.examples import to_examples, with_scores
|
|
18
|
+
from strop.evaluation.models import EvaluationRecord, NewEvaluation
|
|
19
|
+
from strop.evaluation.parsing import load_output_schema
|
|
20
|
+
from strop.evaluation.runner import Progress, PromptRunner
|
|
21
|
+
from strop.llm.client import LLMClient
|
|
22
|
+
from strop.llm.factory import build_client
|
|
23
|
+
from strop.log import LOGGER_NAME
|
|
24
|
+
from strop.metrics.suite import MetricSuite
|
|
25
|
+
from strop.prompts.versions import PromptVersion
|
|
26
|
+
from strop.services.metrics import build_suite, judge_model
|
|
27
|
+
from strop.services.partitions import PartitionedData, load_partitions
|
|
28
|
+
from strop.services.project import Project
|
|
29
|
+
from strop.storage.cache import ResponseCache
|
|
30
|
+
from strop.storage.datasets import DatasetRepository
|
|
31
|
+
from strop.storage.evaluations import EvaluationRepository
|
|
32
|
+
|
|
33
|
+
LOG = logging.getLogger(LOGGER_NAME)
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
@dataclass(frozen=True)
|
|
37
|
+
class RunTag:
|
|
38
|
+
"""Labels an evaluation: what it is for, free-form details, and its optimization run."""
|
|
39
|
+
|
|
40
|
+
kind: str
|
|
41
|
+
details: dict[str, Any] = field(default_factory=dict)
|
|
42
|
+
run_id: int | None = None
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
@dataclass(frozen=True)
|
|
46
|
+
class Evaluator:
|
|
47
|
+
"""Runs prompt versions against partitions and persists each evaluation.
|
|
48
|
+
|
|
49
|
+
The test partition is refused unless the evaluator was created with ``allow_test``;
|
|
50
|
+
only the final ``strop test`` step does that.
|
|
51
|
+
"""
|
|
52
|
+
|
|
53
|
+
project: Project
|
|
54
|
+
partitions: PartitionedData
|
|
55
|
+
suite: MetricSuite
|
|
56
|
+
client: LLMClient
|
|
57
|
+
repository: EvaluationRepository
|
|
58
|
+
budget: BudgetTracker
|
|
59
|
+
target: ModelSpec
|
|
60
|
+
schema: dict[str, Any] | None = None
|
|
61
|
+
allow_test: bool = False
|
|
62
|
+
|
|
63
|
+
def fingerprint(self) -> str:
|
|
64
|
+
"""Hash of every setting that changes what an evaluation measures."""
|
|
65
|
+
config = self.project.config
|
|
66
|
+
payload = {
|
|
67
|
+
"target": self.target.model_dump(mode="json"),
|
|
68
|
+
"output": config.output.model_dump(mode="json"),
|
|
69
|
+
"metrics": config.metrics.model_dump(mode="json"),
|
|
70
|
+
}
|
|
71
|
+
return hashlib.sha256(json.dumps(payload, sort_keys=True).encode()).hexdigest()
|
|
72
|
+
|
|
73
|
+
def reusable(self, version: PromptVersion, partition: Partition) -> EvaluationRecord | None:
|
|
74
|
+
"""A stored full-partition evaluation of ``version`` made under the same settings."""
|
|
75
|
+
expected = len(self.partitions.get(partition))
|
|
76
|
+
for record in self.repository.find(
|
|
77
|
+
prompt_number=version.number, partition=partition, limit=50
|
|
78
|
+
):
|
|
79
|
+
if (
|
|
80
|
+
record.split_id == self.partitions.split_id
|
|
81
|
+
and record.sample_count == expected
|
|
82
|
+
and not record.details.get("sampled")
|
|
83
|
+
and record.details.get("fingerprint") == self.fingerprint()
|
|
84
|
+
):
|
|
85
|
+
return record
|
|
86
|
+
return None
|
|
87
|
+
|
|
88
|
+
def rows(self, partition: Partition) -> list[Row]:
|
|
89
|
+
if partition is Partition.TEST and not self.allow_test:
|
|
90
|
+
raise DataError("the test partition is locked; it is only used by 'strop test'")
|
|
91
|
+
return self.partitions.get(partition)
|
|
92
|
+
|
|
93
|
+
async def evaluate(
|
|
94
|
+
self,
|
|
95
|
+
version: PromptVersion,
|
|
96
|
+
partition: Partition,
|
|
97
|
+
tag: RunTag,
|
|
98
|
+
*,
|
|
99
|
+
rows: Sequence[Row] | None = None,
|
|
100
|
+
progress: Progress | None = None,
|
|
101
|
+
) -> EvaluationRecord:
|
|
102
|
+
"""Evaluate ``version`` on ``partition`` (or a subset of its rows) and store the result."""
|
|
103
|
+
selected = list(rows) if rows is not None else self.rows(partition)
|
|
104
|
+
if rows is not None:
|
|
105
|
+
self.rows(partition) # enforces the test lock for explicit subsets too
|
|
106
|
+
config = self.project.config
|
|
107
|
+
runner = PromptRunner(
|
|
108
|
+
self.client,
|
|
109
|
+
self.target,
|
|
110
|
+
config.output,
|
|
111
|
+
self.schema,
|
|
112
|
+
config.execution,
|
|
113
|
+
self.budget,
|
|
114
|
+
)
|
|
115
|
+
outcomes = await runner.run(version.template, selected, progress)
|
|
116
|
+
examples = to_examples(selected, outcomes)
|
|
117
|
+
await self.suite.prepare(examples)
|
|
118
|
+
report = self.suite.evaluate(examples)
|
|
119
|
+
return self.repository.save(
|
|
120
|
+
NewEvaluation(
|
|
121
|
+
prompt_number=version.number,
|
|
122
|
+
split_id=self.partitions.split_id,
|
|
123
|
+
partition=partition,
|
|
124
|
+
kind=tag.kind,
|
|
125
|
+
report=report,
|
|
126
|
+
outcomes=with_scores(outcomes, examples, self.suite),
|
|
127
|
+
details={
|
|
128
|
+
"model": self.target.model,
|
|
129
|
+
"fingerprint": self.fingerprint(),
|
|
130
|
+
**tag.details,
|
|
131
|
+
},
|
|
132
|
+
run_id=tag.run_id,
|
|
133
|
+
)
|
|
134
|
+
)
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
def build_evaluator(
|
|
138
|
+
project: Project,
|
|
139
|
+
conn: sqlite3.Connection,
|
|
140
|
+
*,
|
|
141
|
+
use_cache: bool = True,
|
|
142
|
+
allow_test: bool = False,
|
|
143
|
+
budget: BudgetTracker | None = None,
|
|
144
|
+
) -> Evaluator:
|
|
145
|
+
"""Assemble an evaluator from the project configuration and an open database."""
|
|
146
|
+
config = project.config
|
|
147
|
+
tracker = budget or BudgetTracker(config.budgets)
|
|
148
|
+
cache = ResponseCache(conn) if use_cache else None
|
|
149
|
+
judge_spec = judge_model(project)
|
|
150
|
+
judge = None
|
|
151
|
+
if judge_spec is not None:
|
|
152
|
+
judge_client = BudgetedClient(build_client(config.execution, cache), tracker)
|
|
153
|
+
judge = (judge_client, judge_spec)
|
|
154
|
+
if judge_spec.model == config.models.target.model:
|
|
155
|
+
LOG.warning(
|
|
156
|
+
"the judge model is the same as the target model; its scores may favor its "
|
|
157
|
+
"own style. Use a different judge model if you can."
|
|
158
|
+
)
|
|
159
|
+
return Evaluator(
|
|
160
|
+
project=project,
|
|
161
|
+
partitions=load_partitions(project, DatasetRepository(conn)),
|
|
162
|
+
suite=build_suite(project, judge),
|
|
163
|
+
client=build_client(config.execution, cache),
|
|
164
|
+
repository=EvaluationRepository(conn),
|
|
165
|
+
budget=tracker,
|
|
166
|
+
target=config.models.resolve(ModelRole.TARGET),
|
|
167
|
+
schema=load_output_schema(project.root, config.output),
|
|
168
|
+
allow_test=allow_test,
|
|
169
|
+
)
|
strop/services/export.py
ADDED
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
"""Writing the exported prompt and report files."""
|
|
2
|
+
|
|
3
|
+
import csv
|
|
4
|
+
import io
|
|
5
|
+
import json
|
|
6
|
+
import sqlite3
|
|
7
|
+
from dataclasses import dataclass
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
from typing import Any
|
|
10
|
+
|
|
11
|
+
from strop.data.models import Partition
|
|
12
|
+
from strop.errors import ConfigError
|
|
13
|
+
from strop.prompts.versions import PromptVersion
|
|
14
|
+
from strop.reporting.prompt_export import PromptFormat, render_prompt
|
|
15
|
+
from strop.reporting.report_markdown import render_markdown
|
|
16
|
+
from strop.services.project import Project
|
|
17
|
+
from strop.services.prompts import ensure_initial_version, resolve_version
|
|
18
|
+
from strop.services.report import build_report
|
|
19
|
+
from strop.storage.evaluations import EvaluationRepository
|
|
20
|
+
from strop.storage.prompts import PromptRepository
|
|
21
|
+
|
|
22
|
+
CSV_COLUMNS = (
|
|
23
|
+
"evaluation_id", "partition", "row_id", "expected", "model_output", "normalized_output",
|
|
24
|
+
"error", "primary_score", "feedback", "latency_s", "total_tokens", "cost_usd",
|
|
25
|
+
) # fmt: skip
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
@dataclass(frozen=True)
|
|
29
|
+
class ExportResult:
|
|
30
|
+
version: PromptVersion
|
|
31
|
+
files: list[Path]
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def export_project( # noqa: PLR0913
|
|
35
|
+
project: Project,
|
|
36
|
+
conn: sqlite3.Connection,
|
|
37
|
+
*,
|
|
38
|
+
ref: str | None,
|
|
39
|
+
out: Path,
|
|
40
|
+
prompt_format: PromptFormat,
|
|
41
|
+
include_results: bool,
|
|
42
|
+
write_prompt: Path | None,
|
|
43
|
+
force: bool,
|
|
44
|
+
) -> ExportResult:
|
|
45
|
+
"""Write the selected prompt, a JSON and Markdown report, and per-example results."""
|
|
46
|
+
prompts = PromptRepository(conn)
|
|
47
|
+
ensure_initial_version(project, prompts)
|
|
48
|
+
version = resolve_version(prompts, ref)
|
|
49
|
+
report = build_report(project, conn, version)
|
|
50
|
+
targets = {
|
|
51
|
+
out / f"prompt_{version.ref}.{prompt_format.value}": render_prompt(version, prompt_format),
|
|
52
|
+
out / "report.json": json.dumps(report, indent=2, ensure_ascii=False) + "\n",
|
|
53
|
+
out / "report.md": render_markdown(report),
|
|
54
|
+
}
|
|
55
|
+
if include_results:
|
|
56
|
+
targets[out / "results.csv"] = _results_csv(conn, version, report)
|
|
57
|
+
if write_prompt is not None:
|
|
58
|
+
targets[write_prompt] = render_prompt(version, PromptFormat.YAML)
|
|
59
|
+
_check_overwrite(list(targets), force)
|
|
60
|
+
for path, text in targets.items():
|
|
61
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
62
|
+
path.write_text(text, encoding="utf-8", newline="")
|
|
63
|
+
return ExportResult(version, list(targets))
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def _check_overwrite(paths: list[Path], force: bool) -> None:
|
|
67
|
+
existing = [p for p in paths if p.exists()]
|
|
68
|
+
if existing and not force:
|
|
69
|
+
names = ", ".join(str(p) for p in existing)
|
|
70
|
+
raise ConfigError(f"would overwrite {names}; pass --force to replace the files")
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def _results_csv(conn: sqlite3.Connection, version: PromptVersion, report: dict[str, Any]) -> str:
|
|
74
|
+
evals = EvaluationRepository(conn)
|
|
75
|
+
buffer = io.StringIO()
|
|
76
|
+
writer = csv.writer(buffer, lineterminator="\n")
|
|
77
|
+
writer.writerow(CSV_COLUMNS)
|
|
78
|
+
primary = str(report["metrics"]["primary"])
|
|
79
|
+
for partition in (Partition.VALIDATION, Partition.TEST):
|
|
80
|
+
found = evals.find(prompt_number=version.number, partition=partition, limit=1)
|
|
81
|
+
for record in found:
|
|
82
|
+
for o in evals.outcomes(record.id):
|
|
83
|
+
writer.writerow(
|
|
84
|
+
[
|
|
85
|
+
record.id, partition.value, o.row_id, o.expected, o.raw_output, o.value,
|
|
86
|
+
o.error, o.scores.get(primary), o.feedback, o.latency_s, o.total_tokens,
|
|
87
|
+
o.cost_usd,
|
|
88
|
+
]
|
|
89
|
+
) # fmt: skip
|
|
90
|
+
return buffer.getvalue()
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
"""The final evaluation on the held-out test partition."""
|
|
2
|
+
|
|
3
|
+
import sqlite3
|
|
4
|
+
from dataclasses import dataclass
|
|
5
|
+
|
|
6
|
+
from strop.data.models import Partition
|
|
7
|
+
from strop.errors import ConfigError, DataError
|
|
8
|
+
from strop.evaluation.models import EvaluationRecord
|
|
9
|
+
from strop.evaluation.runner import Progress
|
|
10
|
+
from strop.prompts.versions import PromptVersion
|
|
11
|
+
from strop.services.evaluator import RunTag, build_evaluator
|
|
12
|
+
from strop.services.project import Project
|
|
13
|
+
from strop.services.prompts import ensure_initial_version, resolve_version
|
|
14
|
+
from strop.storage.prompts import PromptRepository
|
|
15
|
+
|
|
16
|
+
CONFIRM_MESSAGE = (
|
|
17
|
+
"the test partition is the final check. Evaluating it and then changing the prompt makes "
|
|
18
|
+
"the result an optimistic estimate. Run again with --confirm to continue"
|
|
19
|
+
)
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
@dataclass(frozen=True)
|
|
23
|
+
class FinalTest:
|
|
24
|
+
version: PromptVersion
|
|
25
|
+
record: EvaluationRecord
|
|
26
|
+
validation: EvaluationRecord | None
|
|
27
|
+
baseline: tuple[PromptVersion, EvaluationRecord] | None
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
async def run_final_test( # noqa: PLR0913
|
|
31
|
+
project: Project,
|
|
32
|
+
conn: sqlite3.Connection,
|
|
33
|
+
*,
|
|
34
|
+
ref: str | None,
|
|
35
|
+
confirm: bool,
|
|
36
|
+
allow_rerun: bool,
|
|
37
|
+
with_baseline: bool,
|
|
38
|
+
use_cache: bool,
|
|
39
|
+
progress: Progress | None = None,
|
|
40
|
+
) -> FinalTest:
|
|
41
|
+
"""Evaluate a prompt on the test partition, guarded by explicit confirmation."""
|
|
42
|
+
if not confirm:
|
|
43
|
+
raise ConfigError(CONFIRM_MESSAGE)
|
|
44
|
+
prompts = PromptRepository(conn)
|
|
45
|
+
ensure_initial_version(project, prompts)
|
|
46
|
+
version = resolve_version(prompts, ref)
|
|
47
|
+
evaluator = build_evaluator(project, conn, use_cache=use_cache, allow_test=True)
|
|
48
|
+
if not evaluator.rows(Partition.TEST):
|
|
49
|
+
raise DataError("there is no test partition; create one with 'strop data split --test 0.2'")
|
|
50
|
+
used = [
|
|
51
|
+
e
|
|
52
|
+
for e in evaluator.repository.find(partition=Partition.TEST, limit=50)
|
|
53
|
+
if e.split_id == evaluator.partitions.split_id
|
|
54
|
+
]
|
|
55
|
+
if used and not allow_rerun:
|
|
56
|
+
raise ConfigError(
|
|
57
|
+
f"the test partition was already used (evaluation {used[0].id}, "
|
|
58
|
+
f"v{used[0].prompt_number}). Testing again after changing the prompt biases the "
|
|
59
|
+
"result; pass --allow-rerun to continue anyway"
|
|
60
|
+
)
|
|
61
|
+
record = await evaluator.evaluate(version, Partition.TEST, RunTag("test"), progress=progress)
|
|
62
|
+
baseline = None
|
|
63
|
+
initial = prompts.first()
|
|
64
|
+
if with_baseline and initial is not None and initial.number != version.number:
|
|
65
|
+
baseline_record = await evaluator.evaluate(
|
|
66
|
+
initial, Partition.TEST, RunTag("test", {"role": "baseline"}), progress=progress
|
|
67
|
+
)
|
|
68
|
+
baseline = (initial, baseline_record)
|
|
69
|
+
return FinalTest(version, record, evaluator.reusable(version, Partition.VALIDATION), baseline)
|
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
"""Building the metric suite a project asks for."""
|
|
2
|
+
|
|
3
|
+
from strop.config import StropConfig
|
|
4
|
+
from strop.config.enums import ModelRole
|
|
5
|
+
from strop.config.llm import ModelSpec
|
|
6
|
+
from strop.errors import ConfigError
|
|
7
|
+
from strop.llm.client import LLMClient
|
|
8
|
+
from strop.metrics.judge import JudgeMetric
|
|
9
|
+
from strop.metrics.python_metric import METRIC_NAME, PythonMetric
|
|
10
|
+
from strop.metrics.registry import MetricContext, build_metric
|
|
11
|
+
from strop.metrics.suite import MetricSuite
|
|
12
|
+
from strop.metrics.types import Direction, Metric
|
|
13
|
+
from strop.services.project import Project
|
|
14
|
+
|
|
15
|
+
JudgeAccess = tuple[LLMClient, ModelSpec]
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def build_suite(project: Project, judge: JudgeAccess | None = None) -> MetricSuite:
|
|
19
|
+
"""The primary, secondary, rule, custom, judge, and constraint metrics from strop.yaml."""
|
|
20
|
+
config = project.config
|
|
21
|
+
extra: dict[str, Metric] = {}
|
|
22
|
+
if config.metrics.python is not None:
|
|
23
|
+
python = config.metrics.python
|
|
24
|
+
extra[METRIC_NAME] = PythonMetric(
|
|
25
|
+
project.root / python.file,
|
|
26
|
+
timeout_s=python.timeout_s,
|
|
27
|
+
direction=Direction(python.direction),
|
|
28
|
+
)
|
|
29
|
+
for name, spec in config.metrics.judges.items():
|
|
30
|
+
if judge is None:
|
|
31
|
+
raise ConfigError(
|
|
32
|
+
f"judge metric '{name}' needs a judge model; set models.judge.model in "
|
|
33
|
+
"strop.yaml (strop init --judge-model writes it)"
|
|
34
|
+
)
|
|
35
|
+
client, model = judge
|
|
36
|
+
extra[f"judge:{name}"] = JudgeMetric(
|
|
37
|
+
name,
|
|
38
|
+
spec,
|
|
39
|
+
client,
|
|
40
|
+
model,
|
|
41
|
+
config.project.task_description,
|
|
42
|
+
config.execution.max_concurrency,
|
|
43
|
+
)
|
|
44
|
+
validate_metric_names(config)
|
|
45
|
+
context = MetricContext(labels=config.output.labels, rules=config.metrics.rules)
|
|
46
|
+
suite = MetricSuite.from_config(config.metrics, context, extra)
|
|
47
|
+
_check_references(project, suite)
|
|
48
|
+
return suite
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def validate_metric_names(config: StropConfig) -> None:
|
|
52
|
+
"""Fail early, with the list of valid names, if a configured metric does not exist."""
|
|
53
|
+
metrics = config.metrics
|
|
54
|
+
context = MetricContext(labels=config.output.labels, rules=metrics.rules)
|
|
55
|
+
for name in (metrics.primary, *metrics.secondary, *metrics.constraints):
|
|
56
|
+
if name == METRIC_NAME:
|
|
57
|
+
if metrics.python is None:
|
|
58
|
+
raise ConfigError(
|
|
59
|
+
"metric 'custom' needs metrics.python in strop.yaml "
|
|
60
|
+
"(strop init --metric-file writes it)"
|
|
61
|
+
)
|
|
62
|
+
elif name.startswith("judge:"):
|
|
63
|
+
if name.removeprefix("judge:") not in metrics.judges:
|
|
64
|
+
raise ConfigError(
|
|
65
|
+
f"metric '{name}' has no rubric under metrics.judges in strop.yaml "
|
|
66
|
+
"(strop init --rubric-file writes it)"
|
|
67
|
+
)
|
|
68
|
+
else:
|
|
69
|
+
build_metric(name, context)
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def judge_model(project: Project) -> ModelSpec | None:
|
|
73
|
+
"""The judge model if any judge rubric is configured."""
|
|
74
|
+
if not project.config.metrics.judges:
|
|
75
|
+
return None
|
|
76
|
+
return project.config.models.resolve(ModelRole.JUDGE)
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def _check_references(project: Project, suite: MetricSuite) -> None:
|
|
80
|
+
"""Metrics that compare with expected outputs need an expected column."""
|
|
81
|
+
if project.config.dataset.expected_column is not None:
|
|
82
|
+
return
|
|
83
|
+
config = project.config.metrics
|
|
84
|
+
for name in (config.primary, *config.constraints):
|
|
85
|
+
if suite.metric(name).needs_expected:
|
|
86
|
+
raise ConfigError(
|
|
87
|
+
f"metric '{name}' compares outputs with expected values, but no dataset column "
|
|
88
|
+
"has the role 'expected'; set --expected or choose another metric"
|
|
89
|
+
)
|