strop-cli 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- strop/__init__.py +5 -0
- strop/__main__.py +5 -0
- strop/analysis/__init__.py +17 -0
- strop/analysis/classify.py +52 -0
- strop/analysis/evidence.py +52 -0
- strop/analysis/facts.py +74 -0
- strop/analysis/patterns.py +90 -0
- strop/cli/__init__.py +5 -0
- strop/cli/analyze.py +93 -0
- strop/cli/app.py +82 -0
- strop/cli/data.py +111 -0
- strop/cli/evaluate.py +145 -0
- strop/cli/export.py +74 -0
- strop/cli/final.py +106 -0
- strop/cli/init.py +131 -0
- strop/cli/optimize.py +156 -0
- strop/cli/options.py +20 -0
- strop/cli/progress.py +18 -0
- strop/cli/prompts.py +110 -0
- strop/cli/status.py +68 -0
- strop/cli/versions.py +123 -0
- strop/config/__init__.py +16 -0
- strop/config/base.py +9 -0
- strop/config/constraints.py +33 -0
- strop/config/dataset.py +63 -0
- strop/config/enums.py +52 -0
- strop/config/io.py +52 -0
- strop/config/limits.py +19 -0
- strop/config/llm.py +42 -0
- strop/config/metrics.py +86 -0
- strop/config/optimization.py +23 -0
- strop/config/output.py +31 -0
- strop/config/overrides.py +22 -0
- strop/config/root.py +36 -0
- strop/config/rules.py +67 -0
- strop/config/split.py +23 -0
- strop/data/__init__.py +58 -0
- strop/data/loaders.py +85 -0
- strop/data/models.py +44 -0
- strop/data/rows.py +99 -0
- strop/data/splitting.py +103 -0
- strop/env.py +44 -0
- strop/errors.py +59 -0
- strop/evaluation/__init__.py +1 -0
- strop/evaluation/budget.py +67 -0
- strop/evaluation/budgeted.py +23 -0
- strop/evaluation/examples.py +49 -0
- strop/evaluation/models.py +105 -0
- strop/evaluation/parsing.py +96 -0
- strop/evaluation/runner.py +107 -0
- strop/formatting.py +14 -0
- strop/llm/__init__.py +7 -0
- strop/llm/caching.py +28 -0
- strop/llm/client.py +15 -0
- strop/llm/factory.py +14 -0
- strop/llm/litellm_client.py +90 -0
- strop/llm/mock.py +43 -0
- strop/llm/retry.py +44 -0
- strop/llm/structured.py +52 -0
- strop/llm/types.py +59 -0
- strop/log.py +49 -0
- strop/metrics/__init__.py +1 -0
- strop/metrics/basic.py +30 -0
- strop/metrics/classification.py +164 -0
- strop/metrics/judge.py +130 -0
- strop/metrics/python_metric.py +101 -0
- strop/metrics/python_runner.py +65 -0
- strop/metrics/registry.py +115 -0
- strop/metrics/rules.py +55 -0
- strop/metrics/structured.py +109 -0
- strop/metrics/suite.py +109 -0
- strop/metrics/text.py +55 -0
- strop/metrics/types.py +106 -0
- strop/optimization/__init__.py +25 -0
- strop/optimization/acceptance.py +170 -0
- strop/optimization/generator.py +92 -0
- strop/optimization/meta_prompt.py +101 -0
- strop/optimization/types.py +69 -0
- strop/prompts/__init__.py +18 -0
- strop/prompts/diff.py +139 -0
- strop/prompts/files.py +92 -0
- strop/prompts/models.py +51 -0
- strop/prompts/render.py +16 -0
- strop/prompts/template.py +45 -0
- strop/prompts/validation.py +31 -0
- strop/prompts/versions.py +34 -0
- strop/py.typed +0 -0
- strop/reporting/__init__.py +6 -0
- strop/reporting/console.py +20 -0
- strop/reporting/diff_view.py +102 -0
- strop/reporting/evaluation_view.py +141 -0
- strop/reporting/final_view.py +109 -0
- strop/reporting/prompt_export.py +58 -0
- strop/reporting/report_markdown.py +157 -0
- strop/reporting/run_view.py +118 -0
- strop/services/__init__.py +1 -0
- strop/services/analysis.py +124 -0
- strop/services/clean.py +38 -0
- strop/services/compare.py +60 -0
- strop/services/context.py +70 -0
- strop/services/data.py +65 -0
- strop/services/evaluate.py +105 -0
- strop/services/evaluator.py +169 -0
- strop/services/export.py +90 -0
- strop/services/final_test.py +69 -0
- strop/services/metrics.py +89 -0
- strop/services/optimize.py +187 -0
- strop/services/partitions.py +46 -0
- strop/services/project.py +55 -0
- strop/services/prompts.py +76 -0
- strop/services/report.py +194 -0
- strop/services/review.py +43 -0
- strop/services/scaffold.py +79 -0
- strop/services/turn.py +234 -0
- strop/storage/__init__.py +6 -0
- strop/storage/cache.py +45 -0
- strop/storage/database.py +45 -0
- strop/storage/datasets.py +129 -0
- strop/storage/evaluations.py +160 -0
- strop/storage/migrations/001_initial.sql +5 -0
- strop/storage/migrations/002_datasets.sql +27 -0
- strop/storage/migrations/003_prompt_versions.sql +22 -0
- strop/storage/migrations/004_llm_cache.sql +6 -0
- strop/storage/migrations/005_evaluations.sql +37 -0
- strop/storage/migrations/006_runs.sql +52 -0
- strop/storage/migrations/007_example_feedback.sql +2 -0
- strop/storage/migrations/__init__.py +5 -0
- strop/storage/migrator.py +91 -0
- strop/storage/paths.py +30 -0
- strop/storage/prompts.py +96 -0
- strop/storage/run_models.py +75 -0
- strop/storage/runs.py +263 -0
- strop_cli-0.1.0.dist-info/METADATA +242 -0
- strop_cli-0.1.0.dist-info/RECORD +137 -0
- strop_cli-0.1.0.dist-info/WHEEL +4 -0
- strop_cli-0.1.0.dist-info/entry_points.txt +2 -0
- strop_cli-0.1.0.dist-info/licenses/LICENSE +21 -0
strop/services/turn.py
ADDED
|
@@ -0,0 +1,234 @@
|
|
|
1
|
+
"""One optimization turn: analyze, propose, test, and decide."""
|
|
2
|
+
|
|
3
|
+
from dataclasses import dataclass
|
|
4
|
+
|
|
5
|
+
from strop.config.enums import ApprovalMode
|
|
6
|
+
from strop.data.models import Partition
|
|
7
|
+
from strop.evaluation.models import EvaluationRecord
|
|
8
|
+
from strop.formatting import fmt
|
|
9
|
+
from strop.optimization import (
|
|
10
|
+
Candidate,
|
|
11
|
+
HistoryEntry,
|
|
12
|
+
MetricSummary,
|
|
13
|
+
OptimizerView,
|
|
14
|
+
RejectedProposal,
|
|
15
|
+
compare,
|
|
16
|
+
)
|
|
17
|
+
from strop.prompts.versions import PromptVersion
|
|
18
|
+
from strop.services.analysis import TurnAnalysis, analyze_version
|
|
19
|
+
from strop.services.context import LoopContext
|
|
20
|
+
from strop.services.evaluator import RunTag
|
|
21
|
+
from strop.storage.runs import CandidateRecord, NewCandidate, RunRecord
|
|
22
|
+
|
|
23
|
+
HISTORY_LIMIT = 8
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
@dataclass(frozen=True)
|
|
27
|
+
class TurnResult:
|
|
28
|
+
best: PromptVersion
|
|
29
|
+
improved: bool = False
|
|
30
|
+
awaiting: CandidateRecord | None = None
|
|
31
|
+
no_failures: bool = False
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
class TurnRunner:
|
|
35
|
+
def __init__(self, ctx: LoopContext, run: RunRecord) -> None:
|
|
36
|
+
self._ctx = ctx
|
|
37
|
+
self._run = run
|
|
38
|
+
|
|
39
|
+
async def validation_record(self, version: PromptVersion) -> EvaluationRecord:
|
|
40
|
+
"""The validation result for ``version``: a stored one if the settings match."""
|
|
41
|
+
evaluator = self._ctx.evaluator
|
|
42
|
+
found = evaluator.reusable(version, Partition.VALIDATION)
|
|
43
|
+
if found is not None:
|
|
44
|
+
return found
|
|
45
|
+
tag = RunTag("best", run_id=self._run.id)
|
|
46
|
+
return await evaluator.evaluate(version, Partition.VALIDATION, tag)
|
|
47
|
+
|
|
48
|
+
async def run(self, number: int, best: PromptVersion) -> TurnResult:
|
|
49
|
+
ctx, run = self._ctx, self._run
|
|
50
|
+
if not any(t.number == number for t in ctx.runs.turns(run.id)):
|
|
51
|
+
ctx.runs.create_turn(run.id, number, best.number)
|
|
52
|
+
best_record = await self.validation_record(best)
|
|
53
|
+
value = best_record.report.primary_value.value
|
|
54
|
+
ctx.say(f"Turn {number}: {best.ref} scores {best_record.report.primary} {fmt(value)}.")
|
|
55
|
+
|
|
56
|
+
analysis = await analyze_version(
|
|
57
|
+
ctx.evaluator,
|
|
58
|
+
best,
|
|
59
|
+
ctx.settings,
|
|
60
|
+
seed=ctx.evaluator.partitions.seed + number,
|
|
61
|
+
analyzer=(ctx.analyzer_client, ctx.analyzer_spec),
|
|
62
|
+
run_id=run.id,
|
|
63
|
+
)
|
|
64
|
+
ctx.runs.update_turn(
|
|
65
|
+
run.id,
|
|
66
|
+
number,
|
|
67
|
+
train_evaluation_id=analysis.train_record.id,
|
|
68
|
+
analysis=analysis.to_json(),
|
|
69
|
+
)
|
|
70
|
+
if analysis.facts.failed == 0:
|
|
71
|
+
ctx.say(" No failures on the training examples; nothing to improve.")
|
|
72
|
+
ctx.runs.update_turn(run.id, number, status="done", best_after=best.number)
|
|
73
|
+
return TurnResult(best, no_failures=True)
|
|
74
|
+
ctx.say(
|
|
75
|
+
f" Analyzed {analysis.facts.failed} failed training example(s); "
|
|
76
|
+
f"{len(analysis.patterns)} pattern hypothesis(es)."
|
|
77
|
+
)
|
|
78
|
+
view = self._view(best_record, best, analysis)
|
|
79
|
+
existing = len(ctx.runs.candidates(run.id, number))
|
|
80
|
+
for attempt in range(existing, ctx.settings.candidates_per_turn):
|
|
81
|
+
await self._attempt(number, attempt, best, best_record, view)
|
|
82
|
+
return self._conclude(number, best)
|
|
83
|
+
|
|
84
|
+
def _view(
|
|
85
|
+
self, record: EvaluationRecord, best: PromptVersion, analysis: TurnAnalysis
|
|
86
|
+
) -> OptimizerView:
|
|
87
|
+
ctx = self._ctx
|
|
88
|
+
config = ctx.project.config
|
|
89
|
+
primary = record.report.primary_value
|
|
90
|
+
history = tuple(
|
|
91
|
+
HistoryEntry(c.change_summary or c.reason, c.status, c.reason, c.primary_delta)
|
|
92
|
+
for c in ctx.runs.recent(HISTORY_LIMIT)
|
|
93
|
+
)
|
|
94
|
+
return OptimizerView(
|
|
95
|
+
task=config.project.task_description,
|
|
96
|
+
output=config.output,
|
|
97
|
+
current=best.template,
|
|
98
|
+
metrics=MetricSummary(
|
|
99
|
+
record.report.primary,
|
|
100
|
+
primary.value,
|
|
101
|
+
ctx.evaluator.suite.direction(record.report.primary),
|
|
102
|
+
config.metrics.constraints,
|
|
103
|
+
primary.details.get("per_class", {}),
|
|
104
|
+
),
|
|
105
|
+
facts=analysis.facts,
|
|
106
|
+
evidence=analysis.evidence,
|
|
107
|
+
patterns=analysis.patterns,
|
|
108
|
+
history=history,
|
|
109
|
+
instructions=ctx.settings.instructions,
|
|
110
|
+
evidence_chars=ctx.settings.evidence_chars,
|
|
111
|
+
)
|
|
112
|
+
|
|
113
|
+
async def _attempt(
|
|
114
|
+
self,
|
|
115
|
+
number: int,
|
|
116
|
+
attempt: int,
|
|
117
|
+
best: PromptVersion,
|
|
118
|
+
best_record: EvaluationRecord,
|
|
119
|
+
view: OptimizerView,
|
|
120
|
+
) -> None:
|
|
121
|
+
ctx, run = self._ctx, self._run
|
|
122
|
+
proposal = await ctx.generator.propose(view, (number - 1) + attempt)
|
|
123
|
+
if isinstance(proposal, RejectedProposal):
|
|
124
|
+
ctx.say(f" Candidate {attempt + 1}: discarded ({proposal.reason})")
|
|
125
|
+
ctx.runs.add_candidate(
|
|
126
|
+
NewCandidate(
|
|
127
|
+
run.id,
|
|
128
|
+
number,
|
|
129
|
+
best.number,
|
|
130
|
+
"invalid",
|
|
131
|
+
reason=proposal.reason,
|
|
132
|
+
raw_response=proposal.raw_response,
|
|
133
|
+
)
|
|
134
|
+
)
|
|
135
|
+
return
|
|
136
|
+
duplicate = ctx.prompts.find_by_hash(proposal.template.content_hash)
|
|
137
|
+
if duplicate is not None:
|
|
138
|
+
reason = f"identical to the existing version {duplicate.ref}"
|
|
139
|
+
ctx.say(f" Candidate {attempt + 1}: discarded ({reason})")
|
|
140
|
+
ctx.runs.add_candidate(
|
|
141
|
+
_candidate_row(run.id, number, best, proposal, "invalid", reason)
|
|
142
|
+
)
|
|
143
|
+
return
|
|
144
|
+
version = ctx.prompts.add(
|
|
145
|
+
proposal.template,
|
|
146
|
+
parent_number=best.number,
|
|
147
|
+
change_summary=proposal.change_summary,
|
|
148
|
+
reason=f"run {run.id}, turn {number}",
|
|
149
|
+
source="optimizer",
|
|
150
|
+
)
|
|
151
|
+
record = await ctx.evaluator.evaluate(
|
|
152
|
+
version, Partition.VALIDATION, RunTag("candidate", {"turn": number}, run.id)
|
|
153
|
+
)
|
|
154
|
+
decision = compare(
|
|
155
|
+
best_record,
|
|
156
|
+
ctx.evaluator.repository.outcomes(best_record.id),
|
|
157
|
+
record,
|
|
158
|
+
ctx.evaluator.repository.outcomes(record.id),
|
|
159
|
+
ctx.rules,
|
|
160
|
+
ctx.evaluator.suite.direction(best_record.report.primary),
|
|
161
|
+
)
|
|
162
|
+
ctx.runs.add_candidate(
|
|
163
|
+
NewCandidate(
|
|
164
|
+
run.id,
|
|
165
|
+
number,
|
|
166
|
+
best.number,
|
|
167
|
+
"evaluated",
|
|
168
|
+
reason=decision.summary,
|
|
169
|
+
version_number=version.number,
|
|
170
|
+
change_summary=proposal.change_summary,
|
|
171
|
+
patterns=list(proposal.target_patterns),
|
|
172
|
+
evidence=list(proposal.evidence_ids),
|
|
173
|
+
tradeoffs=list(proposal.tradeoffs),
|
|
174
|
+
assumptions=list(proposal.assumptions),
|
|
175
|
+
raw_response=proposal.raw_response,
|
|
176
|
+
evaluation_id=record.id,
|
|
177
|
+
passes=decision.passes,
|
|
178
|
+
primary_delta=decision.primary_delta,
|
|
179
|
+
wins=decision.wins,
|
|
180
|
+
losses=decision.losses,
|
|
181
|
+
ties=decision.ties,
|
|
182
|
+
)
|
|
183
|
+
)
|
|
184
|
+
value = record.report.primary_value.value
|
|
185
|
+
verdict = "passes all rules" if decision.passes else f"rejected: {decision.summary}"
|
|
186
|
+
ctx.say(
|
|
187
|
+
f" Candidate {version.ref}: {proposal.change_summary} "
|
|
188
|
+
f"[{record.report.primary} {fmt(value)}, wins {decision.wins} / losses "
|
|
189
|
+
f"{decision.losses}] {verdict}"
|
|
190
|
+
)
|
|
191
|
+
|
|
192
|
+
def _conclude(self, number: int, best: PromptVersion) -> TurnResult:
|
|
193
|
+
ctx, run = self._ctx, self._run
|
|
194
|
+
evaluated = [c for c in ctx.runs.candidates(run.id, number) if c.status == "evaluated"]
|
|
195
|
+
passers = [c for c in evaluated if c.passes]
|
|
196
|
+
winner = max(passers, key=lambda c: c.primary_delta or 0.0, default=None)
|
|
197
|
+
for candidate in evaluated:
|
|
198
|
+
if candidate is winner:
|
|
199
|
+
continue
|
|
200
|
+
reason = candidate.reason
|
|
201
|
+
if candidate.passes and winner is not None:
|
|
202
|
+
reason = f"passed all rules, but v{winner.version_number} was chosen this turn"
|
|
203
|
+
ctx.runs.set_candidate_status(candidate.id, "rejected", reason)
|
|
204
|
+
if winner is None or winner.version_number is None:
|
|
205
|
+
ctx.say(" No candidate was accepted; the best prompt is unchanged.")
|
|
206
|
+
ctx.runs.update_turn(run.id, number, status="done", best_after=best.number)
|
|
207
|
+
return TurnResult(best)
|
|
208
|
+
if ctx.settings.approval is ApprovalMode.MANUAL:
|
|
209
|
+
ctx.runs.set_candidate_status(
|
|
210
|
+
winner.id, "pending", f"{winner.reason}; waiting for approval"
|
|
211
|
+
)
|
|
212
|
+
ctx.runs.update_turn(run.id, number, status="awaiting_approval")
|
|
213
|
+
return TurnResult(best, awaiting=ctx.runs.get_candidate(winner.id))
|
|
214
|
+
ctx.runs.set_candidate_status(winner.id, "accepted", winner.reason)
|
|
215
|
+
ctx.prompts.set_best(winner.version_number)
|
|
216
|
+
ctx.runs.update_turn(run.id, number, status="done", best_after=winner.version_number)
|
|
217
|
+
ctx.say(f" Accepted v{winner.version_number} as the new best prompt.")
|
|
218
|
+
return TurnResult(ctx.prompts.get(winner.version_number), improved=True)
|
|
219
|
+
|
|
220
|
+
|
|
221
|
+
def _candidate_row(
|
|
222
|
+
run_id: int, turn: int, best: PromptVersion, proposal: Candidate, status: str, reason: str
|
|
223
|
+
) -> NewCandidate:
|
|
224
|
+
return NewCandidate(
|
|
225
|
+
run_id,
|
|
226
|
+
turn,
|
|
227
|
+
best.number,
|
|
228
|
+
status,
|
|
229
|
+
reason=reason,
|
|
230
|
+
change_summary=proposal.change_summary,
|
|
231
|
+
patterns=list(proposal.target_patterns),
|
|
232
|
+
evidence=list(proposal.evidence_ids),
|
|
233
|
+
raw_response=proposal.raw_response,
|
|
234
|
+
)
|
strop/storage/cache.py
ADDED
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
"""Persistent cache of model responses."""
|
|
2
|
+
|
|
3
|
+
import hashlib
|
|
4
|
+
import json
|
|
5
|
+
import sqlite3
|
|
6
|
+
|
|
7
|
+
from strop.llm.types import LLMRequest, LLMResponse
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def cache_key(request: LLMRequest) -> str:
|
|
11
|
+
"""Hash of the model, messages, and generation settings that determine a response."""
|
|
12
|
+
payload = {
|
|
13
|
+
"model": request.model,
|
|
14
|
+
"messages": [[m.role, m.content] for m in request.messages],
|
|
15
|
+
"temperature": request.temperature,
|
|
16
|
+
"max_tokens": request.max_tokens,
|
|
17
|
+
"params": request.params,
|
|
18
|
+
}
|
|
19
|
+
text = json.dumps(payload, sort_keys=True, ensure_ascii=False, default=str)
|
|
20
|
+
return hashlib.sha256(text.encode("utf-8")).hexdigest()
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
class ResponseCache:
|
|
24
|
+
def __init__(self, conn: sqlite3.Connection) -> None:
|
|
25
|
+
self._conn = conn
|
|
26
|
+
|
|
27
|
+
def get(self, key: str) -> LLMResponse | None:
|
|
28
|
+
row = self._conn.execute(
|
|
29
|
+
"SELECT response_json FROM llm_cache WHERE key = ?", (key,)
|
|
30
|
+
).fetchone()
|
|
31
|
+
if row is None:
|
|
32
|
+
return None
|
|
33
|
+
return LLMResponse.model_validate_json(row["response_json"]).model_copy(
|
|
34
|
+
update={"cached": True, "latency_s": 0.0}
|
|
35
|
+
)
|
|
36
|
+
|
|
37
|
+
def put(self, key: str, response: LLMResponse) -> None:
|
|
38
|
+
self._conn.execute(
|
|
39
|
+
"INSERT OR REPLACE INTO llm_cache (key, response_json) VALUES (?, ?)",
|
|
40
|
+
(key, response.model_dump_json()),
|
|
41
|
+
)
|
|
42
|
+
|
|
43
|
+
def clear(self) -> int:
|
|
44
|
+
"""Delete every cached response and return how many there were."""
|
|
45
|
+
return self._conn.execute("DELETE FROM llm_cache").rowcount
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
"""SQLite connection handling."""
|
|
2
|
+
|
|
3
|
+
import sqlite3
|
|
4
|
+
from collections.abc import Iterator
|
|
5
|
+
from contextlib import contextmanager
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
|
|
8
|
+
from strop.errors import StorageError
|
|
9
|
+
from strop.storage.migrator import migrate
|
|
10
|
+
from strop.storage.paths import ProjectPaths
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def connect(path: Path | str) -> sqlite3.Connection:
|
|
14
|
+
"""Open a connection in autocommit mode with foreign keys enforced.
|
|
15
|
+
|
|
16
|
+
Transactions are explicit: use ``transaction`` for multi-statement writes.
|
|
17
|
+
"""
|
|
18
|
+
try:
|
|
19
|
+
conn = sqlite3.connect(path, isolation_level=None)
|
|
20
|
+
conn.row_factory = sqlite3.Row
|
|
21
|
+
conn.execute("PRAGMA foreign_keys = ON")
|
|
22
|
+
conn.execute("PRAGMA journal_mode = WAL")
|
|
23
|
+
except sqlite3.Error as exc:
|
|
24
|
+
raise StorageError(f"cannot open the project database at {path}: {exc}") from exc
|
|
25
|
+
return conn
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
@contextmanager
|
|
29
|
+
def transaction(conn: sqlite3.Connection) -> Iterator[sqlite3.Connection]:
|
|
30
|
+
"""Run a block atomically: commit on success, roll back on any exception."""
|
|
31
|
+
conn.execute("BEGIN IMMEDIATE")
|
|
32
|
+
try:
|
|
33
|
+
yield conn
|
|
34
|
+
except BaseException:
|
|
35
|
+
conn.execute("ROLLBACK")
|
|
36
|
+
raise
|
|
37
|
+
conn.execute("COMMIT")
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def open_database(paths: ProjectPaths) -> sqlite3.Connection:
|
|
41
|
+
"""Create the state directories if needed, open the database, and migrate it."""
|
|
42
|
+
paths.ensure()
|
|
43
|
+
conn = connect(paths.database)
|
|
44
|
+
migrate(conn)
|
|
45
|
+
return conn
|
|
@@ -0,0 +1,129 @@
|
|
|
1
|
+
"""Persistence for dataset versions and split assignments."""
|
|
2
|
+
|
|
3
|
+
import sqlite3
|
|
4
|
+
from collections.abc import Mapping
|
|
5
|
+
from dataclasses import dataclass
|
|
6
|
+
|
|
7
|
+
from strop.data.models import Partition
|
|
8
|
+
from strop.storage.database import transaction
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
@dataclass(frozen=True, slots=True)
|
|
12
|
+
class SplitRecord:
|
|
13
|
+
id: int
|
|
14
|
+
dataset_id: int
|
|
15
|
+
content_hash: str
|
|
16
|
+
skip_invalid: bool
|
|
17
|
+
seed: int
|
|
18
|
+
strategy: str
|
|
19
|
+
config_json: str
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
@dataclass(frozen=True, slots=True)
|
|
23
|
+
class DatasetInfo:
|
|
24
|
+
id: int
|
|
25
|
+
content_hash: str
|
|
26
|
+
source_path: str
|
|
27
|
+
row_count: int
|
|
28
|
+
skipped_rows: int
|
|
29
|
+
skip_invalid: bool
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
class DatasetRepository:
|
|
33
|
+
def __init__(self, conn: sqlite3.Connection) -> None:
|
|
34
|
+
self._conn = conn
|
|
35
|
+
|
|
36
|
+
def save_dataset(
|
|
37
|
+
self,
|
|
38
|
+
*,
|
|
39
|
+
content_hash: str,
|
|
40
|
+
source_path: str,
|
|
41
|
+
row_count: int,
|
|
42
|
+
skipped_rows: int,
|
|
43
|
+
skip_invalid: bool,
|
|
44
|
+
) -> int:
|
|
45
|
+
"""Record a dataset version (idempotent) and return its id."""
|
|
46
|
+
self._conn.execute(
|
|
47
|
+
"INSERT OR IGNORE INTO datasets "
|
|
48
|
+
"(content_hash, source_path, row_count, skipped_rows, skip_invalid) "
|
|
49
|
+
"VALUES (?, ?, ?, ?, ?)",
|
|
50
|
+
(content_hash, source_path, row_count, skipped_rows, int(skip_invalid)),
|
|
51
|
+
)
|
|
52
|
+
row = self._conn.execute(
|
|
53
|
+
"SELECT id FROM datasets WHERE content_hash = ? AND skip_invalid = ?",
|
|
54
|
+
(content_hash, int(skip_invalid)),
|
|
55
|
+
).fetchone()
|
|
56
|
+
return int(row["id"])
|
|
57
|
+
|
|
58
|
+
def create_split(
|
|
59
|
+
self,
|
|
60
|
+
*,
|
|
61
|
+
dataset_id: int,
|
|
62
|
+
seed: int,
|
|
63
|
+
strategy: str,
|
|
64
|
+
config_json: str,
|
|
65
|
+
assignments: Mapping[str, Partition],
|
|
66
|
+
) -> int:
|
|
67
|
+
"""Store a split and all its row assignments atomically; return the split id."""
|
|
68
|
+
with transaction(self._conn):
|
|
69
|
+
cursor = self._conn.execute(
|
|
70
|
+
"INSERT INTO splits (dataset_id, seed, strategy, config_json) VALUES (?, ?, ?, ?)",
|
|
71
|
+
(dataset_id, seed, strategy, config_json),
|
|
72
|
+
)
|
|
73
|
+
split_id = int(cursor.lastrowid or 0)
|
|
74
|
+
self._conn.executemany(
|
|
75
|
+
"INSERT INTO split_assignments (split_id, row_id, partition) VALUES (?, ?, ?)",
|
|
76
|
+
[(split_id, row_id, part.value) for row_id, part in assignments.items()],
|
|
77
|
+
)
|
|
78
|
+
return split_id
|
|
79
|
+
|
|
80
|
+
def find_split(self, dataset_id: int, config_json: str) -> int | None:
|
|
81
|
+
row = self._conn.execute(
|
|
82
|
+
"SELECT id FROM splits WHERE dataset_id = ? AND config_json = ? ORDER BY id DESC",
|
|
83
|
+
(dataset_id, config_json),
|
|
84
|
+
).fetchone()
|
|
85
|
+
return int(row["id"]) if row else None
|
|
86
|
+
|
|
87
|
+
def latest_split(self) -> SplitRecord | None:
|
|
88
|
+
row = self._conn.execute(
|
|
89
|
+
"SELECT s.id, s.dataset_id, d.content_hash, d.skip_invalid, s.seed, s.strategy, "
|
|
90
|
+
"s.config_json FROM splits s JOIN datasets d ON d.id = s.dataset_id "
|
|
91
|
+
"ORDER BY s.id DESC LIMIT 1"
|
|
92
|
+
).fetchone()
|
|
93
|
+
if row is None:
|
|
94
|
+
return None
|
|
95
|
+
return SplitRecord(
|
|
96
|
+
row["id"],
|
|
97
|
+
row["dataset_id"],
|
|
98
|
+
row["content_hash"],
|
|
99
|
+
bool(row["skip_invalid"]),
|
|
100
|
+
row["seed"],
|
|
101
|
+
row["strategy"],
|
|
102
|
+
row["config_json"],
|
|
103
|
+
)
|
|
104
|
+
|
|
105
|
+
def assignments(self, split_id: int) -> dict[str, Partition]:
|
|
106
|
+
rows = self._conn.execute(
|
|
107
|
+
"SELECT row_id, partition FROM split_assignments WHERE split_id = ?", (split_id,)
|
|
108
|
+
).fetchall()
|
|
109
|
+
return {r["row_id"]: Partition(r["partition"]) for r in rows}
|
|
110
|
+
|
|
111
|
+
def partition_counts(self, split_id: int) -> dict[Partition, int]:
|
|
112
|
+
rows = self._conn.execute(
|
|
113
|
+
"SELECT partition, COUNT(*) AS n FROM split_assignments WHERE split_id = ? "
|
|
114
|
+
"GROUP BY partition",
|
|
115
|
+
(split_id,),
|
|
116
|
+
).fetchall()
|
|
117
|
+
counts = {Partition(r["partition"]): r["n"] for r in rows}
|
|
118
|
+
return {partition: counts.get(partition, 0) for partition in Partition}
|
|
119
|
+
|
|
120
|
+
def dataset_info(self, dataset_id: int) -> DatasetInfo:
|
|
121
|
+
row = self._conn.execute("SELECT * FROM datasets WHERE id = ?", (dataset_id,)).fetchone()
|
|
122
|
+
return DatasetInfo(
|
|
123
|
+
row["id"],
|
|
124
|
+
row["content_hash"],
|
|
125
|
+
row["source_path"],
|
|
126
|
+
row["row_count"],
|
|
127
|
+
row["skipped_rows"],
|
|
128
|
+
bool(row["skip_invalid"]),
|
|
129
|
+
)
|
|
@@ -0,0 +1,160 @@
|
|
|
1
|
+
"""Persistence for evaluations and their per-example results."""
|
|
2
|
+
|
|
3
|
+
import json
|
|
4
|
+
import sqlite3
|
|
5
|
+
from dataclasses import asdict
|
|
6
|
+
from typing import Any
|
|
7
|
+
|
|
8
|
+
from strop.data.models import Partition
|
|
9
|
+
from strop.errors import StorageError
|
|
10
|
+
from strop.evaluation.models import (
|
|
11
|
+
EvaluationRecord,
|
|
12
|
+
ExampleOutcome,
|
|
13
|
+
NewEvaluation,
|
|
14
|
+
UsageSummary,
|
|
15
|
+
summarize_usage,
|
|
16
|
+
)
|
|
17
|
+
from strop.metrics.suite import MetricReport
|
|
18
|
+
from strop.storage.database import transaction
|
|
19
|
+
|
|
20
|
+
_OUTCOME_COLUMNS = (
|
|
21
|
+
"row_id, expected, raw_output, value, parsed_json, error, detail, finish_reason, latency_s, "
|
|
22
|
+
"prompt_tokens, completion_tokens, total_tokens, cost_usd, cached, scores_json, feedback"
|
|
23
|
+
)
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
_INSERT_OUTCOME = (
|
|
27
|
+
f"INSERT INTO example_results (evaluation_id, {_OUTCOME_COLUMNS}) "
|
|
28
|
+
f"VALUES ({', '.join('?' * 17)})"
|
|
29
|
+
)
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
class EvaluationRepository:
|
|
33
|
+
def __init__(self, conn: sqlite3.Connection) -> None:
|
|
34
|
+
self._conn = conn
|
|
35
|
+
|
|
36
|
+
def save(self, new: NewEvaluation) -> EvaluationRecord:
|
|
37
|
+
"""Store an evaluation and all of its example results atomically."""
|
|
38
|
+
usage = summarize_usage(new.outcomes)
|
|
39
|
+
with transaction(self._conn):
|
|
40
|
+
cursor = self._conn.execute(
|
|
41
|
+
"INSERT INTO evaluations (prompt_number, split_id, partition, kind, run_id, "
|
|
42
|
+
"metrics_json, usage_json, details_json, sample_count, error_count) "
|
|
43
|
+
"VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
|
|
44
|
+
(
|
|
45
|
+
new.prompt_number,
|
|
46
|
+
new.split_id,
|
|
47
|
+
new.partition.value,
|
|
48
|
+
new.kind,
|
|
49
|
+
new.run_id,
|
|
50
|
+
json.dumps(new.report.to_dict()),
|
|
51
|
+
json.dumps(asdict(usage)),
|
|
52
|
+
json.dumps(new.details),
|
|
53
|
+
len(new.outcomes),
|
|
54
|
+
sum(o.failed for o in new.outcomes),
|
|
55
|
+
),
|
|
56
|
+
)
|
|
57
|
+
evaluation_id = int(cursor.lastrowid or 0)
|
|
58
|
+
self._conn.executemany(
|
|
59
|
+
_INSERT_OUTCOME, [self._row(evaluation_id, o) for o in new.outcomes]
|
|
60
|
+
)
|
|
61
|
+
return self.get(evaluation_id)
|
|
62
|
+
|
|
63
|
+
def get(self, evaluation_id: int) -> EvaluationRecord:
|
|
64
|
+
row = self._conn.execute(
|
|
65
|
+
"SELECT * FROM evaluations WHERE id = ?", (evaluation_id,)
|
|
66
|
+
).fetchone()
|
|
67
|
+
if row is None:
|
|
68
|
+
raise StorageError(f"evaluation {evaluation_id} does not exist")
|
|
69
|
+
return _record(row)
|
|
70
|
+
|
|
71
|
+
def find(
|
|
72
|
+
self,
|
|
73
|
+
*,
|
|
74
|
+
prompt_number: int | None = None,
|
|
75
|
+
partition: Partition | None = None,
|
|
76
|
+
kind: str | None = None,
|
|
77
|
+
limit: int = 20,
|
|
78
|
+
) -> list[EvaluationRecord]:
|
|
79
|
+
"""Matching evaluations, newest first."""
|
|
80
|
+
clauses, args = [], []
|
|
81
|
+
for column, value in (
|
|
82
|
+
("prompt_number", prompt_number),
|
|
83
|
+
("partition", partition.value if partition else None),
|
|
84
|
+
("kind", kind),
|
|
85
|
+
):
|
|
86
|
+
if value is not None:
|
|
87
|
+
clauses.append(f"{column} = ?")
|
|
88
|
+
args.append(value)
|
|
89
|
+
where = f"WHERE {' AND '.join(clauses)}" if clauses else ""
|
|
90
|
+
rows = self._conn.execute(
|
|
91
|
+
f"SELECT * FROM evaluations {where} ORDER BY id DESC LIMIT ?", (*args, limit)
|
|
92
|
+
).fetchall()
|
|
93
|
+
return [_record(row) for row in rows]
|
|
94
|
+
|
|
95
|
+
def outcomes(self, evaluation_id: int) -> list[ExampleOutcome]:
|
|
96
|
+
rows = self._conn.execute(
|
|
97
|
+
f"SELECT {_OUTCOME_COLUMNS} FROM example_results WHERE evaluation_id = ? "
|
|
98
|
+
"ORDER BY rowid",
|
|
99
|
+
(evaluation_id,),
|
|
100
|
+
).fetchall()
|
|
101
|
+
return [_outcome(row) for row in rows]
|
|
102
|
+
|
|
103
|
+
@staticmethod
|
|
104
|
+
def _row(evaluation_id: int, o: ExampleOutcome) -> tuple[Any, ...]:
|
|
105
|
+
return (
|
|
106
|
+
evaluation_id,
|
|
107
|
+
o.row_id,
|
|
108
|
+
o.expected,
|
|
109
|
+
o.raw_output,
|
|
110
|
+
o.value,
|
|
111
|
+
json.dumps(o.parsed) if o.parsed is not None else None,
|
|
112
|
+
o.error,
|
|
113
|
+
o.detail,
|
|
114
|
+
o.finish_reason,
|
|
115
|
+
o.latency_s,
|
|
116
|
+
o.prompt_tokens,
|
|
117
|
+
o.completion_tokens,
|
|
118
|
+
o.total_tokens,
|
|
119
|
+
o.cost_usd,
|
|
120
|
+
int(o.cached),
|
|
121
|
+
json.dumps(o.scores),
|
|
122
|
+
o.feedback,
|
|
123
|
+
)
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
def _record(row: sqlite3.Row) -> EvaluationRecord:
|
|
127
|
+
return EvaluationRecord(
|
|
128
|
+
id=row["id"],
|
|
129
|
+
prompt_number=row["prompt_number"],
|
|
130
|
+
split_id=row["split_id"],
|
|
131
|
+
partition=Partition(row["partition"]),
|
|
132
|
+
kind=row["kind"],
|
|
133
|
+
report=MetricReport.from_dict(json.loads(row["metrics_json"])),
|
|
134
|
+
usage=UsageSummary(**json.loads(row["usage_json"])),
|
|
135
|
+
sample_count=row["sample_count"],
|
|
136
|
+
error_count=row["error_count"],
|
|
137
|
+
created_at=row["created_at"],
|
|
138
|
+
details=json.loads(row["details_json"]),
|
|
139
|
+
)
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
def _outcome(row: sqlite3.Row) -> ExampleOutcome:
|
|
143
|
+
return ExampleOutcome(
|
|
144
|
+
row_id=row["row_id"],
|
|
145
|
+
expected=row["expected"],
|
|
146
|
+
raw_output=row["raw_output"],
|
|
147
|
+
value=row["value"],
|
|
148
|
+
parsed=json.loads(row["parsed_json"]) if row["parsed_json"] else None,
|
|
149
|
+
error=row["error"],
|
|
150
|
+
detail=row["detail"],
|
|
151
|
+
finish_reason=row["finish_reason"],
|
|
152
|
+
latency_s=row["latency_s"],
|
|
153
|
+
prompt_tokens=row["prompt_tokens"],
|
|
154
|
+
completion_tokens=row["completion_tokens"],
|
|
155
|
+
total_tokens=row["total_tokens"],
|
|
156
|
+
cost_usd=row["cost_usd"],
|
|
157
|
+
cached=bool(row["cached"]),
|
|
158
|
+
scores=json.loads(row["scores_json"]),
|
|
159
|
+
feedback=row["feedback"],
|
|
160
|
+
)
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
-- Dataset versions (identified by file content) and the splits made from them.
|
|
2
|
+
CREATE TABLE datasets (
|
|
3
|
+
id INTEGER PRIMARY KEY,
|
|
4
|
+
content_hash TEXT NOT NULL,
|
|
5
|
+
source_path TEXT NOT NULL,
|
|
6
|
+
row_count INTEGER NOT NULL,
|
|
7
|
+
skipped_rows INTEGER NOT NULL DEFAULT 0,
|
|
8
|
+
skip_invalid INTEGER NOT NULL DEFAULT 0,
|
|
9
|
+
created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
|
10
|
+
UNIQUE (content_hash, skip_invalid)
|
|
11
|
+
);
|
|
12
|
+
|
|
13
|
+
CREATE TABLE splits (
|
|
14
|
+
id INTEGER PRIMARY KEY,
|
|
15
|
+
dataset_id INTEGER NOT NULL REFERENCES datasets (id),
|
|
16
|
+
seed INTEGER NOT NULL,
|
|
17
|
+
strategy TEXT NOT NULL CHECK (strategy IN ('stratified', 'random')),
|
|
18
|
+
config_json TEXT NOT NULL,
|
|
19
|
+
created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP
|
|
20
|
+
);
|
|
21
|
+
|
|
22
|
+
CREATE TABLE split_assignments (
|
|
23
|
+
split_id INTEGER NOT NULL REFERENCES splits (id) ON DELETE CASCADE,
|
|
24
|
+
row_id TEXT NOT NULL,
|
|
25
|
+
partition TEXT NOT NULL CHECK (partition IN ('train', 'validation', 'test')),
|
|
26
|
+
PRIMARY KEY (split_id, row_id)
|
|
27
|
+
);
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
-- Immutable prompt versions. A revision never overwrites its parent.
|
|
2
|
+
CREATE TABLE prompt_versions (
|
|
3
|
+
id INTEGER PRIMARY KEY,
|
|
4
|
+
number INTEGER NOT NULL UNIQUE,
|
|
5
|
+
parent_number INTEGER REFERENCES prompt_versions (number),
|
|
6
|
+
content_json TEXT NOT NULL,
|
|
7
|
+
content_hash TEXT NOT NULL,
|
|
8
|
+
change_summary TEXT NOT NULL DEFAULT '',
|
|
9
|
+
reason TEXT NOT NULL DEFAULT '',
|
|
10
|
+
source TEXT NOT NULL CHECK (source IN ('user', 'optimizer')),
|
|
11
|
+
created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP
|
|
12
|
+
);
|
|
13
|
+
|
|
14
|
+
CREATE TRIGGER prompt_versions_no_update BEFORE UPDATE ON prompt_versions
|
|
15
|
+
BEGIN
|
|
16
|
+
SELECT RAISE(ABORT, 'prompt versions are immutable');
|
|
17
|
+
END;
|
|
18
|
+
|
|
19
|
+
CREATE TRIGGER prompt_versions_no_delete BEFORE DELETE ON prompt_versions
|
|
20
|
+
BEGIN
|
|
21
|
+
SELECT RAISE(ABORT, 'prompt versions are immutable');
|
|
22
|
+
END;
|