strop-cli 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (137) hide show
  1. strop/__init__.py +5 -0
  2. strop/__main__.py +5 -0
  3. strop/analysis/__init__.py +17 -0
  4. strop/analysis/classify.py +52 -0
  5. strop/analysis/evidence.py +52 -0
  6. strop/analysis/facts.py +74 -0
  7. strop/analysis/patterns.py +90 -0
  8. strop/cli/__init__.py +5 -0
  9. strop/cli/analyze.py +93 -0
  10. strop/cli/app.py +82 -0
  11. strop/cli/data.py +111 -0
  12. strop/cli/evaluate.py +145 -0
  13. strop/cli/export.py +74 -0
  14. strop/cli/final.py +106 -0
  15. strop/cli/init.py +131 -0
  16. strop/cli/optimize.py +156 -0
  17. strop/cli/options.py +20 -0
  18. strop/cli/progress.py +18 -0
  19. strop/cli/prompts.py +110 -0
  20. strop/cli/status.py +68 -0
  21. strop/cli/versions.py +123 -0
  22. strop/config/__init__.py +16 -0
  23. strop/config/base.py +9 -0
  24. strop/config/constraints.py +33 -0
  25. strop/config/dataset.py +63 -0
  26. strop/config/enums.py +52 -0
  27. strop/config/io.py +52 -0
  28. strop/config/limits.py +19 -0
  29. strop/config/llm.py +42 -0
  30. strop/config/metrics.py +86 -0
  31. strop/config/optimization.py +23 -0
  32. strop/config/output.py +31 -0
  33. strop/config/overrides.py +22 -0
  34. strop/config/root.py +36 -0
  35. strop/config/rules.py +67 -0
  36. strop/config/split.py +23 -0
  37. strop/data/__init__.py +58 -0
  38. strop/data/loaders.py +85 -0
  39. strop/data/models.py +44 -0
  40. strop/data/rows.py +99 -0
  41. strop/data/splitting.py +103 -0
  42. strop/env.py +44 -0
  43. strop/errors.py +59 -0
  44. strop/evaluation/__init__.py +1 -0
  45. strop/evaluation/budget.py +67 -0
  46. strop/evaluation/budgeted.py +23 -0
  47. strop/evaluation/examples.py +49 -0
  48. strop/evaluation/models.py +105 -0
  49. strop/evaluation/parsing.py +96 -0
  50. strop/evaluation/runner.py +107 -0
  51. strop/formatting.py +14 -0
  52. strop/llm/__init__.py +7 -0
  53. strop/llm/caching.py +28 -0
  54. strop/llm/client.py +15 -0
  55. strop/llm/factory.py +14 -0
  56. strop/llm/litellm_client.py +90 -0
  57. strop/llm/mock.py +43 -0
  58. strop/llm/retry.py +44 -0
  59. strop/llm/structured.py +52 -0
  60. strop/llm/types.py +59 -0
  61. strop/log.py +49 -0
  62. strop/metrics/__init__.py +1 -0
  63. strop/metrics/basic.py +30 -0
  64. strop/metrics/classification.py +164 -0
  65. strop/metrics/judge.py +130 -0
  66. strop/metrics/python_metric.py +101 -0
  67. strop/metrics/python_runner.py +65 -0
  68. strop/metrics/registry.py +115 -0
  69. strop/metrics/rules.py +55 -0
  70. strop/metrics/structured.py +109 -0
  71. strop/metrics/suite.py +109 -0
  72. strop/metrics/text.py +55 -0
  73. strop/metrics/types.py +106 -0
  74. strop/optimization/__init__.py +25 -0
  75. strop/optimization/acceptance.py +170 -0
  76. strop/optimization/generator.py +92 -0
  77. strop/optimization/meta_prompt.py +101 -0
  78. strop/optimization/types.py +69 -0
  79. strop/prompts/__init__.py +18 -0
  80. strop/prompts/diff.py +139 -0
  81. strop/prompts/files.py +92 -0
  82. strop/prompts/models.py +51 -0
  83. strop/prompts/render.py +16 -0
  84. strop/prompts/template.py +45 -0
  85. strop/prompts/validation.py +31 -0
  86. strop/prompts/versions.py +34 -0
  87. strop/py.typed +0 -0
  88. strop/reporting/__init__.py +6 -0
  89. strop/reporting/console.py +20 -0
  90. strop/reporting/diff_view.py +102 -0
  91. strop/reporting/evaluation_view.py +141 -0
  92. strop/reporting/final_view.py +109 -0
  93. strop/reporting/prompt_export.py +58 -0
  94. strop/reporting/report_markdown.py +157 -0
  95. strop/reporting/run_view.py +118 -0
  96. strop/services/__init__.py +1 -0
  97. strop/services/analysis.py +124 -0
  98. strop/services/clean.py +38 -0
  99. strop/services/compare.py +60 -0
  100. strop/services/context.py +70 -0
  101. strop/services/data.py +65 -0
  102. strop/services/evaluate.py +105 -0
  103. strop/services/evaluator.py +169 -0
  104. strop/services/export.py +90 -0
  105. strop/services/final_test.py +69 -0
  106. strop/services/metrics.py +89 -0
  107. strop/services/optimize.py +187 -0
  108. strop/services/partitions.py +46 -0
  109. strop/services/project.py +55 -0
  110. strop/services/prompts.py +76 -0
  111. strop/services/report.py +194 -0
  112. strop/services/review.py +43 -0
  113. strop/services/scaffold.py +79 -0
  114. strop/services/turn.py +234 -0
  115. strop/storage/__init__.py +6 -0
  116. strop/storage/cache.py +45 -0
  117. strop/storage/database.py +45 -0
  118. strop/storage/datasets.py +129 -0
  119. strop/storage/evaluations.py +160 -0
  120. strop/storage/migrations/001_initial.sql +5 -0
  121. strop/storage/migrations/002_datasets.sql +27 -0
  122. strop/storage/migrations/003_prompt_versions.sql +22 -0
  123. strop/storage/migrations/004_llm_cache.sql +6 -0
  124. strop/storage/migrations/005_evaluations.sql +37 -0
  125. strop/storage/migrations/006_runs.sql +52 -0
  126. strop/storage/migrations/007_example_feedback.sql +2 -0
  127. strop/storage/migrations/__init__.py +5 -0
  128. strop/storage/migrator.py +91 -0
  129. strop/storage/paths.py +30 -0
  130. strop/storage/prompts.py +96 -0
  131. strop/storage/run_models.py +75 -0
  132. strop/storage/runs.py +263 -0
  133. strop_cli-0.1.0.dist-info/METADATA +242 -0
  134. strop_cli-0.1.0.dist-info/RECORD +137 -0
  135. strop_cli-0.1.0.dist-info/WHEEL +4 -0
  136. strop_cli-0.1.0.dist-info/entry_points.txt +2 -0
  137. strop_cli-0.1.0.dist-info/licenses/LICENSE +21 -0
@@ -0,0 +1,105 @@
1
+ """The ``baseline`` and ``eval`` workflows."""
2
+
3
+ import random
4
+ import sqlite3
5
+ from collections.abc import Sequence
6
+ from dataclasses import dataclass, replace
7
+
8
+ from strop.data.models import Partition, Row
9
+ from strop.evaluation.models import EvaluationRecord
10
+ from strop.evaluation.runner import Progress
11
+ from strop.llm.factory import build_client
12
+ from strop.prompts.versions import PromptVersion
13
+ from strop.services.evaluator import RunTag, build_evaluator
14
+ from strop.services.project import Project
15
+ from strop.services.prompts import ensure_initial_version, resolve_version
16
+ from strop.storage.prompts import PromptRepository
17
+
18
+
19
+ @dataclass(frozen=True)
20
+ class BaselineOutcome:
21
+ version: PromptVersion
22
+ records: list[EvaluationRecord]
23
+ train: EvaluationRecord | None
24
+ noise: float | None
25
+
26
+ @property
27
+ def reference(self) -> EvaluationRecord:
28
+ """The first validation run; later repeats only measure run-to-run noise."""
29
+ return self.records[0]
30
+
31
+
32
+ def sample_rows(rows: Sequence[Row], size: int, seed: int) -> list[Row]:
33
+ """A reproducible random subset of ``rows`` of at most ``size`` rows."""
34
+ if size >= len(rows):
35
+ return list(rows)
36
+ chosen = random.Random(seed).sample(sorted(rows, key=lambda r: r.id), size)
37
+ return sorted(chosen, key=lambda r: r.id)
38
+
39
+
40
+ async def run_baseline(
41
+ project: Project,
42
+ conn: sqlite3.Connection,
43
+ *,
44
+ repeats: int,
45
+ sample: int | None,
46
+ with_train: bool,
47
+ use_cache: bool,
48
+ progress: Progress | None = None,
49
+ ) -> BaselineOutcome:
50
+ """Evaluate the initial prompt (v1) on validation, optionally repeated and on train too.
51
+
52
+ Repeats after the first bypass the response cache, so they measure real run-to-run noise.
53
+ """
54
+ version = ensure_initial_version(project, PromptRepository(conn))
55
+ evaluator = build_evaluator(project, conn, use_cache=use_cache)
56
+ rows = evaluator.rows(Partition.VALIDATION)
57
+ if sample is not None:
58
+ rows = sample_rows(rows, sample, evaluator.partitions.seed)
59
+ records = []
60
+ for repeat in range(1, repeats + 1):
61
+ runner = evaluator
62
+ if repeat > 1:
63
+ runner = replace(evaluator, client=build_client(project.config.execution, None))
64
+ details = {"repeat": repeat, "sampled": sample is not None}
65
+ tag = RunTag("baseline", details)
66
+ records.append(
67
+ await runner.evaluate(version, Partition.VALIDATION, tag, rows=rows, progress=progress)
68
+ )
69
+ train = None
70
+ if with_train:
71
+ train = await evaluator.evaluate(
72
+ version, Partition.TRAIN, RunTag("baseline"), progress=progress
73
+ )
74
+ return BaselineOutcome(version, records, train, _noise(records))
75
+
76
+
77
+ async def run_eval(
78
+ project: Project,
79
+ conn: sqlite3.Connection,
80
+ *,
81
+ ref: str | None,
82
+ partition: Partition,
83
+ sample: int | None,
84
+ use_cache: bool,
85
+ progress: Progress | None = None,
86
+ ) -> tuple[PromptVersion, EvaluationRecord]:
87
+ """Score one prompt version on a partition without changing anything."""
88
+ repo = PromptRepository(conn)
89
+ ensure_initial_version(project, repo)
90
+ version = resolve_version(repo, ref)
91
+ evaluator = build_evaluator(project, conn, use_cache=use_cache)
92
+ rows = evaluator.rows(partition)
93
+ if sample is not None:
94
+ rows = sample_rows(rows, sample, evaluator.partitions.seed)
95
+ tag = RunTag("eval", {"sampled": sample is not None})
96
+ record = await evaluator.evaluate(version, partition, tag, rows=rows, progress=progress)
97
+ return version, record
98
+
99
+
100
+ def _noise(records: Sequence[EvaluationRecord]) -> float | None:
101
+ values = [r.report.primary_value.value for r in records]
102
+ known = [v for v in values if v is not None]
103
+ if len(records) < 2 or len(known) != len(values):
104
+ return None
105
+ return max(known) - min(known)
@@ -0,0 +1,169 @@
1
+ """Evaluating prompt versions on dataset partitions and storing the results."""
2
+
3
+ import hashlib
4
+ import json
5
+ import logging
6
+ import sqlite3
7
+ from collections.abc import Sequence
8
+ from dataclasses import dataclass, field
9
+ from typing import Any
10
+
11
+ from strop.config.enums import ModelRole
12
+ from strop.config.llm import ModelSpec
13
+ from strop.data.models import Partition, Row
14
+ from strop.errors import DataError
15
+ from strop.evaluation.budget import BudgetTracker
16
+ from strop.evaluation.budgeted import BudgetedClient
17
+ from strop.evaluation.examples import to_examples, with_scores
18
+ from strop.evaluation.models import EvaluationRecord, NewEvaluation
19
+ from strop.evaluation.parsing import load_output_schema
20
+ from strop.evaluation.runner import Progress, PromptRunner
21
+ from strop.llm.client import LLMClient
22
+ from strop.llm.factory import build_client
23
+ from strop.log import LOGGER_NAME
24
+ from strop.metrics.suite import MetricSuite
25
+ from strop.prompts.versions import PromptVersion
26
+ from strop.services.metrics import build_suite, judge_model
27
+ from strop.services.partitions import PartitionedData, load_partitions
28
+ from strop.services.project import Project
29
+ from strop.storage.cache import ResponseCache
30
+ from strop.storage.datasets import DatasetRepository
31
+ from strop.storage.evaluations import EvaluationRepository
32
+
33
+ LOG = logging.getLogger(LOGGER_NAME)
34
+
35
+
36
+ @dataclass(frozen=True)
37
+ class RunTag:
38
+ """Labels an evaluation: what it is for, free-form details, and its optimization run."""
39
+
40
+ kind: str
41
+ details: dict[str, Any] = field(default_factory=dict)
42
+ run_id: int | None = None
43
+
44
+
45
+ @dataclass(frozen=True)
46
+ class Evaluator:
47
+ """Runs prompt versions against partitions and persists each evaluation.
48
+
49
+ The test partition is refused unless the evaluator was created with ``allow_test``;
50
+ only the final ``strop test`` step does that.
51
+ """
52
+
53
+ project: Project
54
+ partitions: PartitionedData
55
+ suite: MetricSuite
56
+ client: LLMClient
57
+ repository: EvaluationRepository
58
+ budget: BudgetTracker
59
+ target: ModelSpec
60
+ schema: dict[str, Any] | None = None
61
+ allow_test: bool = False
62
+
63
+ def fingerprint(self) -> str:
64
+ """Hash of every setting that changes what an evaluation measures."""
65
+ config = self.project.config
66
+ payload = {
67
+ "target": self.target.model_dump(mode="json"),
68
+ "output": config.output.model_dump(mode="json"),
69
+ "metrics": config.metrics.model_dump(mode="json"),
70
+ }
71
+ return hashlib.sha256(json.dumps(payload, sort_keys=True).encode()).hexdigest()
72
+
73
+ def reusable(self, version: PromptVersion, partition: Partition) -> EvaluationRecord | None:
74
+ """A stored full-partition evaluation of ``version`` made under the same settings."""
75
+ expected = len(self.partitions.get(partition))
76
+ for record in self.repository.find(
77
+ prompt_number=version.number, partition=partition, limit=50
78
+ ):
79
+ if (
80
+ record.split_id == self.partitions.split_id
81
+ and record.sample_count == expected
82
+ and not record.details.get("sampled")
83
+ and record.details.get("fingerprint") == self.fingerprint()
84
+ ):
85
+ return record
86
+ return None
87
+
88
+ def rows(self, partition: Partition) -> list[Row]:
89
+ if partition is Partition.TEST and not self.allow_test:
90
+ raise DataError("the test partition is locked; it is only used by 'strop test'")
91
+ return self.partitions.get(partition)
92
+
93
+ async def evaluate(
94
+ self,
95
+ version: PromptVersion,
96
+ partition: Partition,
97
+ tag: RunTag,
98
+ *,
99
+ rows: Sequence[Row] | None = None,
100
+ progress: Progress | None = None,
101
+ ) -> EvaluationRecord:
102
+ """Evaluate ``version`` on ``partition`` (or a subset of its rows) and store the result."""
103
+ selected = list(rows) if rows is not None else self.rows(partition)
104
+ if rows is not None:
105
+ self.rows(partition) # enforces the test lock for explicit subsets too
106
+ config = self.project.config
107
+ runner = PromptRunner(
108
+ self.client,
109
+ self.target,
110
+ config.output,
111
+ self.schema,
112
+ config.execution,
113
+ self.budget,
114
+ )
115
+ outcomes = await runner.run(version.template, selected, progress)
116
+ examples = to_examples(selected, outcomes)
117
+ await self.suite.prepare(examples)
118
+ report = self.suite.evaluate(examples)
119
+ return self.repository.save(
120
+ NewEvaluation(
121
+ prompt_number=version.number,
122
+ split_id=self.partitions.split_id,
123
+ partition=partition,
124
+ kind=tag.kind,
125
+ report=report,
126
+ outcomes=with_scores(outcomes, examples, self.suite),
127
+ details={
128
+ "model": self.target.model,
129
+ "fingerprint": self.fingerprint(),
130
+ **tag.details,
131
+ },
132
+ run_id=tag.run_id,
133
+ )
134
+ )
135
+
136
+
137
+ def build_evaluator(
138
+ project: Project,
139
+ conn: sqlite3.Connection,
140
+ *,
141
+ use_cache: bool = True,
142
+ allow_test: bool = False,
143
+ budget: BudgetTracker | None = None,
144
+ ) -> Evaluator:
145
+ """Assemble an evaluator from the project configuration and an open database."""
146
+ config = project.config
147
+ tracker = budget or BudgetTracker(config.budgets)
148
+ cache = ResponseCache(conn) if use_cache else None
149
+ judge_spec = judge_model(project)
150
+ judge = None
151
+ if judge_spec is not None:
152
+ judge_client = BudgetedClient(build_client(config.execution, cache), tracker)
153
+ judge = (judge_client, judge_spec)
154
+ if judge_spec.model == config.models.target.model:
155
+ LOG.warning(
156
+ "the judge model is the same as the target model; its scores may favor its "
157
+ "own style. Use a different judge model if you can."
158
+ )
159
+ return Evaluator(
160
+ project=project,
161
+ partitions=load_partitions(project, DatasetRepository(conn)),
162
+ suite=build_suite(project, judge),
163
+ client=build_client(config.execution, cache),
164
+ repository=EvaluationRepository(conn),
165
+ budget=tracker,
166
+ target=config.models.resolve(ModelRole.TARGET),
167
+ schema=load_output_schema(project.root, config.output),
168
+ allow_test=allow_test,
169
+ )
@@ -0,0 +1,90 @@
1
+ """Writing the exported prompt and report files."""
2
+
3
+ import csv
4
+ import io
5
+ import json
6
+ import sqlite3
7
+ from dataclasses import dataclass
8
+ from pathlib import Path
9
+ from typing import Any
10
+
11
+ from strop.data.models import Partition
12
+ from strop.errors import ConfigError
13
+ from strop.prompts.versions import PromptVersion
14
+ from strop.reporting.prompt_export import PromptFormat, render_prompt
15
+ from strop.reporting.report_markdown import render_markdown
16
+ from strop.services.project import Project
17
+ from strop.services.prompts import ensure_initial_version, resolve_version
18
+ from strop.services.report import build_report
19
+ from strop.storage.evaluations import EvaluationRepository
20
+ from strop.storage.prompts import PromptRepository
21
+
22
+ CSV_COLUMNS = (
23
+ "evaluation_id", "partition", "row_id", "expected", "model_output", "normalized_output",
24
+ "error", "primary_score", "feedback", "latency_s", "total_tokens", "cost_usd",
25
+ ) # fmt: skip
26
+
27
+
28
+ @dataclass(frozen=True)
29
+ class ExportResult:
30
+ version: PromptVersion
31
+ files: list[Path]
32
+
33
+
34
+ def export_project( # noqa: PLR0913
35
+ project: Project,
36
+ conn: sqlite3.Connection,
37
+ *,
38
+ ref: str | None,
39
+ out: Path,
40
+ prompt_format: PromptFormat,
41
+ include_results: bool,
42
+ write_prompt: Path | None,
43
+ force: bool,
44
+ ) -> ExportResult:
45
+ """Write the selected prompt, a JSON and Markdown report, and per-example results."""
46
+ prompts = PromptRepository(conn)
47
+ ensure_initial_version(project, prompts)
48
+ version = resolve_version(prompts, ref)
49
+ report = build_report(project, conn, version)
50
+ targets = {
51
+ out / f"prompt_{version.ref}.{prompt_format.value}": render_prompt(version, prompt_format),
52
+ out / "report.json": json.dumps(report, indent=2, ensure_ascii=False) + "\n",
53
+ out / "report.md": render_markdown(report),
54
+ }
55
+ if include_results:
56
+ targets[out / "results.csv"] = _results_csv(conn, version, report)
57
+ if write_prompt is not None:
58
+ targets[write_prompt] = render_prompt(version, PromptFormat.YAML)
59
+ _check_overwrite(list(targets), force)
60
+ for path, text in targets.items():
61
+ path.parent.mkdir(parents=True, exist_ok=True)
62
+ path.write_text(text, encoding="utf-8", newline="")
63
+ return ExportResult(version, list(targets))
64
+
65
+
66
+ def _check_overwrite(paths: list[Path], force: bool) -> None:
67
+ existing = [p for p in paths if p.exists()]
68
+ if existing and not force:
69
+ names = ", ".join(str(p) for p in existing)
70
+ raise ConfigError(f"would overwrite {names}; pass --force to replace the files")
71
+
72
+
73
+ def _results_csv(conn: sqlite3.Connection, version: PromptVersion, report: dict[str, Any]) -> str:
74
+ evals = EvaluationRepository(conn)
75
+ buffer = io.StringIO()
76
+ writer = csv.writer(buffer, lineterminator="\n")
77
+ writer.writerow(CSV_COLUMNS)
78
+ primary = str(report["metrics"]["primary"])
79
+ for partition in (Partition.VALIDATION, Partition.TEST):
80
+ found = evals.find(prompt_number=version.number, partition=partition, limit=1)
81
+ for record in found:
82
+ for o in evals.outcomes(record.id):
83
+ writer.writerow(
84
+ [
85
+ record.id, partition.value, o.row_id, o.expected, o.raw_output, o.value,
86
+ o.error, o.scores.get(primary), o.feedback, o.latency_s, o.total_tokens,
87
+ o.cost_usd,
88
+ ]
89
+ ) # fmt: skip
90
+ return buffer.getvalue()
@@ -0,0 +1,69 @@
1
+ """The final evaluation on the held-out test partition."""
2
+
3
+ import sqlite3
4
+ from dataclasses import dataclass
5
+
6
+ from strop.data.models import Partition
7
+ from strop.errors import ConfigError, DataError
8
+ from strop.evaluation.models import EvaluationRecord
9
+ from strop.evaluation.runner import Progress
10
+ from strop.prompts.versions import PromptVersion
11
+ from strop.services.evaluator import RunTag, build_evaluator
12
+ from strop.services.project import Project
13
+ from strop.services.prompts import ensure_initial_version, resolve_version
14
+ from strop.storage.prompts import PromptRepository
15
+
16
+ CONFIRM_MESSAGE = (
17
+ "the test partition is the final check. Evaluating it and then changing the prompt makes "
18
+ "the result an optimistic estimate. Run again with --confirm to continue"
19
+ )
20
+
21
+
22
+ @dataclass(frozen=True)
23
+ class FinalTest:
24
+ version: PromptVersion
25
+ record: EvaluationRecord
26
+ validation: EvaluationRecord | None
27
+ baseline: tuple[PromptVersion, EvaluationRecord] | None
28
+
29
+
30
+ async def run_final_test( # noqa: PLR0913
31
+ project: Project,
32
+ conn: sqlite3.Connection,
33
+ *,
34
+ ref: str | None,
35
+ confirm: bool,
36
+ allow_rerun: bool,
37
+ with_baseline: bool,
38
+ use_cache: bool,
39
+ progress: Progress | None = None,
40
+ ) -> FinalTest:
41
+ """Evaluate a prompt on the test partition, guarded by explicit confirmation."""
42
+ if not confirm:
43
+ raise ConfigError(CONFIRM_MESSAGE)
44
+ prompts = PromptRepository(conn)
45
+ ensure_initial_version(project, prompts)
46
+ version = resolve_version(prompts, ref)
47
+ evaluator = build_evaluator(project, conn, use_cache=use_cache, allow_test=True)
48
+ if not evaluator.rows(Partition.TEST):
49
+ raise DataError("there is no test partition; create one with 'strop data split --test 0.2'")
50
+ used = [
51
+ e
52
+ for e in evaluator.repository.find(partition=Partition.TEST, limit=50)
53
+ if e.split_id == evaluator.partitions.split_id
54
+ ]
55
+ if used and not allow_rerun:
56
+ raise ConfigError(
57
+ f"the test partition was already used (evaluation {used[0].id}, "
58
+ f"v{used[0].prompt_number}). Testing again after changing the prompt biases the "
59
+ "result; pass --allow-rerun to continue anyway"
60
+ )
61
+ record = await evaluator.evaluate(version, Partition.TEST, RunTag("test"), progress=progress)
62
+ baseline = None
63
+ initial = prompts.first()
64
+ if with_baseline and initial is not None and initial.number != version.number:
65
+ baseline_record = await evaluator.evaluate(
66
+ initial, Partition.TEST, RunTag("test", {"role": "baseline"}), progress=progress
67
+ )
68
+ baseline = (initial, baseline_record)
69
+ return FinalTest(version, record, evaluator.reusable(version, Partition.VALIDATION), baseline)
@@ -0,0 +1,89 @@
1
+ """Building the metric suite a project asks for."""
2
+
3
+ from strop.config import StropConfig
4
+ from strop.config.enums import ModelRole
5
+ from strop.config.llm import ModelSpec
6
+ from strop.errors import ConfigError
7
+ from strop.llm.client import LLMClient
8
+ from strop.metrics.judge import JudgeMetric
9
+ from strop.metrics.python_metric import METRIC_NAME, PythonMetric
10
+ from strop.metrics.registry import MetricContext, build_metric
11
+ from strop.metrics.suite import MetricSuite
12
+ from strop.metrics.types import Direction, Metric
13
+ from strop.services.project import Project
14
+
15
+ JudgeAccess = tuple[LLMClient, ModelSpec]
16
+
17
+
18
+ def build_suite(project: Project, judge: JudgeAccess | None = None) -> MetricSuite:
19
+ """The primary, secondary, rule, custom, judge, and constraint metrics from strop.yaml."""
20
+ config = project.config
21
+ extra: dict[str, Metric] = {}
22
+ if config.metrics.python is not None:
23
+ python = config.metrics.python
24
+ extra[METRIC_NAME] = PythonMetric(
25
+ project.root / python.file,
26
+ timeout_s=python.timeout_s,
27
+ direction=Direction(python.direction),
28
+ )
29
+ for name, spec in config.metrics.judges.items():
30
+ if judge is None:
31
+ raise ConfigError(
32
+ f"judge metric '{name}' needs a judge model; set models.judge.model in "
33
+ "strop.yaml (strop init --judge-model writes it)"
34
+ )
35
+ client, model = judge
36
+ extra[f"judge:{name}"] = JudgeMetric(
37
+ name,
38
+ spec,
39
+ client,
40
+ model,
41
+ config.project.task_description,
42
+ config.execution.max_concurrency,
43
+ )
44
+ validate_metric_names(config)
45
+ context = MetricContext(labels=config.output.labels, rules=config.metrics.rules)
46
+ suite = MetricSuite.from_config(config.metrics, context, extra)
47
+ _check_references(project, suite)
48
+ return suite
49
+
50
+
51
+ def validate_metric_names(config: StropConfig) -> None:
52
+ """Fail early, with the list of valid names, if a configured metric does not exist."""
53
+ metrics = config.metrics
54
+ context = MetricContext(labels=config.output.labels, rules=metrics.rules)
55
+ for name in (metrics.primary, *metrics.secondary, *metrics.constraints):
56
+ if name == METRIC_NAME:
57
+ if metrics.python is None:
58
+ raise ConfigError(
59
+ "metric 'custom' needs metrics.python in strop.yaml "
60
+ "(strop init --metric-file writes it)"
61
+ )
62
+ elif name.startswith("judge:"):
63
+ if name.removeprefix("judge:") not in metrics.judges:
64
+ raise ConfigError(
65
+ f"metric '{name}' has no rubric under metrics.judges in strop.yaml "
66
+ "(strop init --rubric-file writes it)"
67
+ )
68
+ else:
69
+ build_metric(name, context)
70
+
71
+
72
+ def judge_model(project: Project) -> ModelSpec | None:
73
+ """The judge model if any judge rubric is configured."""
74
+ if not project.config.metrics.judges:
75
+ return None
76
+ return project.config.models.resolve(ModelRole.JUDGE)
77
+
78
+
79
+ def _check_references(project: Project, suite: MetricSuite) -> None:
80
+ """Metrics that compare with expected outputs need an expected column."""
81
+ if project.config.dataset.expected_column is not None:
82
+ return
83
+ config = project.config.metrics
84
+ for name in (config.primary, *config.constraints):
85
+ if suite.metric(name).needs_expected:
86
+ raise ConfigError(
87
+ f"metric '{name}' compares outputs with expected values, but no dataset column "
88
+ "has the role 'expected'; set --expected or choose another metric"
89
+ )