strop-cli 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (137) hide show
  1. strop/__init__.py +5 -0
  2. strop/__main__.py +5 -0
  3. strop/analysis/__init__.py +17 -0
  4. strop/analysis/classify.py +52 -0
  5. strop/analysis/evidence.py +52 -0
  6. strop/analysis/facts.py +74 -0
  7. strop/analysis/patterns.py +90 -0
  8. strop/cli/__init__.py +5 -0
  9. strop/cli/analyze.py +93 -0
  10. strop/cli/app.py +82 -0
  11. strop/cli/data.py +111 -0
  12. strop/cli/evaluate.py +145 -0
  13. strop/cli/export.py +74 -0
  14. strop/cli/final.py +106 -0
  15. strop/cli/init.py +131 -0
  16. strop/cli/optimize.py +156 -0
  17. strop/cli/options.py +20 -0
  18. strop/cli/progress.py +18 -0
  19. strop/cli/prompts.py +110 -0
  20. strop/cli/status.py +68 -0
  21. strop/cli/versions.py +123 -0
  22. strop/config/__init__.py +16 -0
  23. strop/config/base.py +9 -0
  24. strop/config/constraints.py +33 -0
  25. strop/config/dataset.py +63 -0
  26. strop/config/enums.py +52 -0
  27. strop/config/io.py +52 -0
  28. strop/config/limits.py +19 -0
  29. strop/config/llm.py +42 -0
  30. strop/config/metrics.py +86 -0
  31. strop/config/optimization.py +23 -0
  32. strop/config/output.py +31 -0
  33. strop/config/overrides.py +22 -0
  34. strop/config/root.py +36 -0
  35. strop/config/rules.py +67 -0
  36. strop/config/split.py +23 -0
  37. strop/data/__init__.py +58 -0
  38. strop/data/loaders.py +85 -0
  39. strop/data/models.py +44 -0
  40. strop/data/rows.py +99 -0
  41. strop/data/splitting.py +103 -0
  42. strop/env.py +44 -0
  43. strop/errors.py +59 -0
  44. strop/evaluation/__init__.py +1 -0
  45. strop/evaluation/budget.py +67 -0
  46. strop/evaluation/budgeted.py +23 -0
  47. strop/evaluation/examples.py +49 -0
  48. strop/evaluation/models.py +105 -0
  49. strop/evaluation/parsing.py +96 -0
  50. strop/evaluation/runner.py +107 -0
  51. strop/formatting.py +14 -0
  52. strop/llm/__init__.py +7 -0
  53. strop/llm/caching.py +28 -0
  54. strop/llm/client.py +15 -0
  55. strop/llm/factory.py +14 -0
  56. strop/llm/litellm_client.py +90 -0
  57. strop/llm/mock.py +43 -0
  58. strop/llm/retry.py +44 -0
  59. strop/llm/structured.py +52 -0
  60. strop/llm/types.py +59 -0
  61. strop/log.py +49 -0
  62. strop/metrics/__init__.py +1 -0
  63. strop/metrics/basic.py +30 -0
  64. strop/metrics/classification.py +164 -0
  65. strop/metrics/judge.py +130 -0
  66. strop/metrics/python_metric.py +101 -0
  67. strop/metrics/python_runner.py +65 -0
  68. strop/metrics/registry.py +115 -0
  69. strop/metrics/rules.py +55 -0
  70. strop/metrics/structured.py +109 -0
  71. strop/metrics/suite.py +109 -0
  72. strop/metrics/text.py +55 -0
  73. strop/metrics/types.py +106 -0
  74. strop/optimization/__init__.py +25 -0
  75. strop/optimization/acceptance.py +170 -0
  76. strop/optimization/generator.py +92 -0
  77. strop/optimization/meta_prompt.py +101 -0
  78. strop/optimization/types.py +69 -0
  79. strop/prompts/__init__.py +18 -0
  80. strop/prompts/diff.py +139 -0
  81. strop/prompts/files.py +92 -0
  82. strop/prompts/models.py +51 -0
  83. strop/prompts/render.py +16 -0
  84. strop/prompts/template.py +45 -0
  85. strop/prompts/validation.py +31 -0
  86. strop/prompts/versions.py +34 -0
  87. strop/py.typed +0 -0
  88. strop/reporting/__init__.py +6 -0
  89. strop/reporting/console.py +20 -0
  90. strop/reporting/diff_view.py +102 -0
  91. strop/reporting/evaluation_view.py +141 -0
  92. strop/reporting/final_view.py +109 -0
  93. strop/reporting/prompt_export.py +58 -0
  94. strop/reporting/report_markdown.py +157 -0
  95. strop/reporting/run_view.py +118 -0
  96. strop/services/__init__.py +1 -0
  97. strop/services/analysis.py +124 -0
  98. strop/services/clean.py +38 -0
  99. strop/services/compare.py +60 -0
  100. strop/services/context.py +70 -0
  101. strop/services/data.py +65 -0
  102. strop/services/evaluate.py +105 -0
  103. strop/services/evaluator.py +169 -0
  104. strop/services/export.py +90 -0
  105. strop/services/final_test.py +69 -0
  106. strop/services/metrics.py +89 -0
  107. strop/services/optimize.py +187 -0
  108. strop/services/partitions.py +46 -0
  109. strop/services/project.py +55 -0
  110. strop/services/prompts.py +76 -0
  111. strop/services/report.py +194 -0
  112. strop/services/review.py +43 -0
  113. strop/services/scaffold.py +79 -0
  114. strop/services/turn.py +234 -0
  115. strop/storage/__init__.py +6 -0
  116. strop/storage/cache.py +45 -0
  117. strop/storage/database.py +45 -0
  118. strop/storage/datasets.py +129 -0
  119. strop/storage/evaluations.py +160 -0
  120. strop/storage/migrations/001_initial.sql +5 -0
  121. strop/storage/migrations/002_datasets.sql +27 -0
  122. strop/storage/migrations/003_prompt_versions.sql +22 -0
  123. strop/storage/migrations/004_llm_cache.sql +6 -0
  124. strop/storage/migrations/005_evaluations.sql +37 -0
  125. strop/storage/migrations/006_runs.sql +52 -0
  126. strop/storage/migrations/007_example_feedback.sql +2 -0
  127. strop/storage/migrations/__init__.py +5 -0
  128. strop/storage/migrator.py +91 -0
  129. strop/storage/paths.py +30 -0
  130. strop/storage/prompts.py +96 -0
  131. strop/storage/run_models.py +75 -0
  132. strop/storage/runs.py +263 -0
  133. strop_cli-0.1.0.dist-info/METADATA +242 -0
  134. strop_cli-0.1.0.dist-info/RECORD +137 -0
  135. strop_cli-0.1.0.dist-info/WHEEL +4 -0
  136. strop_cli-0.1.0.dist-info/entry_points.txt +2 -0
  137. strop_cli-0.1.0.dist-info/licenses/LICENSE +21 -0
strop/services/turn.py ADDED
@@ -0,0 +1,234 @@
1
+ """One optimization turn: analyze, propose, test, and decide."""
2
+
3
+ from dataclasses import dataclass
4
+
5
+ from strop.config.enums import ApprovalMode
6
+ from strop.data.models import Partition
7
+ from strop.evaluation.models import EvaluationRecord
8
+ from strop.formatting import fmt
9
+ from strop.optimization import (
10
+ Candidate,
11
+ HistoryEntry,
12
+ MetricSummary,
13
+ OptimizerView,
14
+ RejectedProposal,
15
+ compare,
16
+ )
17
+ from strop.prompts.versions import PromptVersion
18
+ from strop.services.analysis import TurnAnalysis, analyze_version
19
+ from strop.services.context import LoopContext
20
+ from strop.services.evaluator import RunTag
21
+ from strop.storage.runs import CandidateRecord, NewCandidate, RunRecord
22
+
23
+ HISTORY_LIMIT = 8
24
+
25
+
26
+ @dataclass(frozen=True)
27
+ class TurnResult:
28
+ best: PromptVersion
29
+ improved: bool = False
30
+ awaiting: CandidateRecord | None = None
31
+ no_failures: bool = False
32
+
33
+
34
+ class TurnRunner:
35
+ def __init__(self, ctx: LoopContext, run: RunRecord) -> None:
36
+ self._ctx = ctx
37
+ self._run = run
38
+
39
+ async def validation_record(self, version: PromptVersion) -> EvaluationRecord:
40
+ """The validation result for ``version``: a stored one if the settings match."""
41
+ evaluator = self._ctx.evaluator
42
+ found = evaluator.reusable(version, Partition.VALIDATION)
43
+ if found is not None:
44
+ return found
45
+ tag = RunTag("best", run_id=self._run.id)
46
+ return await evaluator.evaluate(version, Partition.VALIDATION, tag)
47
+
48
+ async def run(self, number: int, best: PromptVersion) -> TurnResult:
49
+ ctx, run = self._ctx, self._run
50
+ if not any(t.number == number for t in ctx.runs.turns(run.id)):
51
+ ctx.runs.create_turn(run.id, number, best.number)
52
+ best_record = await self.validation_record(best)
53
+ value = best_record.report.primary_value.value
54
+ ctx.say(f"Turn {number}: {best.ref} scores {best_record.report.primary} {fmt(value)}.")
55
+
56
+ analysis = await analyze_version(
57
+ ctx.evaluator,
58
+ best,
59
+ ctx.settings,
60
+ seed=ctx.evaluator.partitions.seed + number,
61
+ analyzer=(ctx.analyzer_client, ctx.analyzer_spec),
62
+ run_id=run.id,
63
+ )
64
+ ctx.runs.update_turn(
65
+ run.id,
66
+ number,
67
+ train_evaluation_id=analysis.train_record.id,
68
+ analysis=analysis.to_json(),
69
+ )
70
+ if analysis.facts.failed == 0:
71
+ ctx.say(" No failures on the training examples; nothing to improve.")
72
+ ctx.runs.update_turn(run.id, number, status="done", best_after=best.number)
73
+ return TurnResult(best, no_failures=True)
74
+ ctx.say(
75
+ f" Analyzed {analysis.facts.failed} failed training example(s); "
76
+ f"{len(analysis.patterns)} pattern hypothesis(es)."
77
+ )
78
+ view = self._view(best_record, best, analysis)
79
+ existing = len(ctx.runs.candidates(run.id, number))
80
+ for attempt in range(existing, ctx.settings.candidates_per_turn):
81
+ await self._attempt(number, attempt, best, best_record, view)
82
+ return self._conclude(number, best)
83
+
84
+ def _view(
85
+ self, record: EvaluationRecord, best: PromptVersion, analysis: TurnAnalysis
86
+ ) -> OptimizerView:
87
+ ctx = self._ctx
88
+ config = ctx.project.config
89
+ primary = record.report.primary_value
90
+ history = tuple(
91
+ HistoryEntry(c.change_summary or c.reason, c.status, c.reason, c.primary_delta)
92
+ for c in ctx.runs.recent(HISTORY_LIMIT)
93
+ )
94
+ return OptimizerView(
95
+ task=config.project.task_description,
96
+ output=config.output,
97
+ current=best.template,
98
+ metrics=MetricSummary(
99
+ record.report.primary,
100
+ primary.value,
101
+ ctx.evaluator.suite.direction(record.report.primary),
102
+ config.metrics.constraints,
103
+ primary.details.get("per_class", {}),
104
+ ),
105
+ facts=analysis.facts,
106
+ evidence=analysis.evidence,
107
+ patterns=analysis.patterns,
108
+ history=history,
109
+ instructions=ctx.settings.instructions,
110
+ evidence_chars=ctx.settings.evidence_chars,
111
+ )
112
+
113
+ async def _attempt(
114
+ self,
115
+ number: int,
116
+ attempt: int,
117
+ best: PromptVersion,
118
+ best_record: EvaluationRecord,
119
+ view: OptimizerView,
120
+ ) -> None:
121
+ ctx, run = self._ctx, self._run
122
+ proposal = await ctx.generator.propose(view, (number - 1) + attempt)
123
+ if isinstance(proposal, RejectedProposal):
124
+ ctx.say(f" Candidate {attempt + 1}: discarded ({proposal.reason})")
125
+ ctx.runs.add_candidate(
126
+ NewCandidate(
127
+ run.id,
128
+ number,
129
+ best.number,
130
+ "invalid",
131
+ reason=proposal.reason,
132
+ raw_response=proposal.raw_response,
133
+ )
134
+ )
135
+ return
136
+ duplicate = ctx.prompts.find_by_hash(proposal.template.content_hash)
137
+ if duplicate is not None:
138
+ reason = f"identical to the existing version {duplicate.ref}"
139
+ ctx.say(f" Candidate {attempt + 1}: discarded ({reason})")
140
+ ctx.runs.add_candidate(
141
+ _candidate_row(run.id, number, best, proposal, "invalid", reason)
142
+ )
143
+ return
144
+ version = ctx.prompts.add(
145
+ proposal.template,
146
+ parent_number=best.number,
147
+ change_summary=proposal.change_summary,
148
+ reason=f"run {run.id}, turn {number}",
149
+ source="optimizer",
150
+ )
151
+ record = await ctx.evaluator.evaluate(
152
+ version, Partition.VALIDATION, RunTag("candidate", {"turn": number}, run.id)
153
+ )
154
+ decision = compare(
155
+ best_record,
156
+ ctx.evaluator.repository.outcomes(best_record.id),
157
+ record,
158
+ ctx.evaluator.repository.outcomes(record.id),
159
+ ctx.rules,
160
+ ctx.evaluator.suite.direction(best_record.report.primary),
161
+ )
162
+ ctx.runs.add_candidate(
163
+ NewCandidate(
164
+ run.id,
165
+ number,
166
+ best.number,
167
+ "evaluated",
168
+ reason=decision.summary,
169
+ version_number=version.number,
170
+ change_summary=proposal.change_summary,
171
+ patterns=list(proposal.target_patterns),
172
+ evidence=list(proposal.evidence_ids),
173
+ tradeoffs=list(proposal.tradeoffs),
174
+ assumptions=list(proposal.assumptions),
175
+ raw_response=proposal.raw_response,
176
+ evaluation_id=record.id,
177
+ passes=decision.passes,
178
+ primary_delta=decision.primary_delta,
179
+ wins=decision.wins,
180
+ losses=decision.losses,
181
+ ties=decision.ties,
182
+ )
183
+ )
184
+ value = record.report.primary_value.value
185
+ verdict = "passes all rules" if decision.passes else f"rejected: {decision.summary}"
186
+ ctx.say(
187
+ f" Candidate {version.ref}: {proposal.change_summary} "
188
+ f"[{record.report.primary} {fmt(value)}, wins {decision.wins} / losses "
189
+ f"{decision.losses}] {verdict}"
190
+ )
191
+
192
+ def _conclude(self, number: int, best: PromptVersion) -> TurnResult:
193
+ ctx, run = self._ctx, self._run
194
+ evaluated = [c for c in ctx.runs.candidates(run.id, number) if c.status == "evaluated"]
195
+ passers = [c for c in evaluated if c.passes]
196
+ winner = max(passers, key=lambda c: c.primary_delta or 0.0, default=None)
197
+ for candidate in evaluated:
198
+ if candidate is winner:
199
+ continue
200
+ reason = candidate.reason
201
+ if candidate.passes and winner is not None:
202
+ reason = f"passed all rules, but v{winner.version_number} was chosen this turn"
203
+ ctx.runs.set_candidate_status(candidate.id, "rejected", reason)
204
+ if winner is None or winner.version_number is None:
205
+ ctx.say(" No candidate was accepted; the best prompt is unchanged.")
206
+ ctx.runs.update_turn(run.id, number, status="done", best_after=best.number)
207
+ return TurnResult(best)
208
+ if ctx.settings.approval is ApprovalMode.MANUAL:
209
+ ctx.runs.set_candidate_status(
210
+ winner.id, "pending", f"{winner.reason}; waiting for approval"
211
+ )
212
+ ctx.runs.update_turn(run.id, number, status="awaiting_approval")
213
+ return TurnResult(best, awaiting=ctx.runs.get_candidate(winner.id))
214
+ ctx.runs.set_candidate_status(winner.id, "accepted", winner.reason)
215
+ ctx.prompts.set_best(winner.version_number)
216
+ ctx.runs.update_turn(run.id, number, status="done", best_after=winner.version_number)
217
+ ctx.say(f" Accepted v{winner.version_number} as the new best prompt.")
218
+ return TurnResult(ctx.prompts.get(winner.version_number), improved=True)
219
+
220
+
221
+ def _candidate_row(
222
+ run_id: int, turn: int, best: PromptVersion, proposal: Candidate, status: str, reason: str
223
+ ) -> NewCandidate:
224
+ return NewCandidate(
225
+ run_id,
226
+ turn,
227
+ best.number,
228
+ status,
229
+ reason=reason,
230
+ change_summary=proposal.change_summary,
231
+ patterns=list(proposal.target_patterns),
232
+ evidence=list(proposal.evidence_ids),
233
+ raw_response=proposal.raw_response,
234
+ )
@@ -0,0 +1,6 @@
1
+ """Local project storage: paths, SQLite connection, and migrations."""
2
+
3
+ from strop.storage.database import connect, open_database, transaction
4
+ from strop.storage.paths import ProjectPaths
5
+
6
+ __all__ = ["ProjectPaths", "connect", "open_database", "transaction"]
strop/storage/cache.py ADDED
@@ -0,0 +1,45 @@
1
+ """Persistent cache of model responses."""
2
+
3
+ import hashlib
4
+ import json
5
+ import sqlite3
6
+
7
+ from strop.llm.types import LLMRequest, LLMResponse
8
+
9
+
10
+ def cache_key(request: LLMRequest) -> str:
11
+ """Hash of the model, messages, and generation settings that determine a response."""
12
+ payload = {
13
+ "model": request.model,
14
+ "messages": [[m.role, m.content] for m in request.messages],
15
+ "temperature": request.temperature,
16
+ "max_tokens": request.max_tokens,
17
+ "params": request.params,
18
+ }
19
+ text = json.dumps(payload, sort_keys=True, ensure_ascii=False, default=str)
20
+ return hashlib.sha256(text.encode("utf-8")).hexdigest()
21
+
22
+
23
+ class ResponseCache:
24
+ def __init__(self, conn: sqlite3.Connection) -> None:
25
+ self._conn = conn
26
+
27
+ def get(self, key: str) -> LLMResponse | None:
28
+ row = self._conn.execute(
29
+ "SELECT response_json FROM llm_cache WHERE key = ?", (key,)
30
+ ).fetchone()
31
+ if row is None:
32
+ return None
33
+ return LLMResponse.model_validate_json(row["response_json"]).model_copy(
34
+ update={"cached": True, "latency_s": 0.0}
35
+ )
36
+
37
+ def put(self, key: str, response: LLMResponse) -> None:
38
+ self._conn.execute(
39
+ "INSERT OR REPLACE INTO llm_cache (key, response_json) VALUES (?, ?)",
40
+ (key, response.model_dump_json()),
41
+ )
42
+
43
+ def clear(self) -> int:
44
+ """Delete every cached response and return how many there were."""
45
+ return self._conn.execute("DELETE FROM llm_cache").rowcount
@@ -0,0 +1,45 @@
1
+ """SQLite connection handling."""
2
+
3
+ import sqlite3
4
+ from collections.abc import Iterator
5
+ from contextlib import contextmanager
6
+ from pathlib import Path
7
+
8
+ from strop.errors import StorageError
9
+ from strop.storage.migrator import migrate
10
+ from strop.storage.paths import ProjectPaths
11
+
12
+
13
+ def connect(path: Path | str) -> sqlite3.Connection:
14
+ """Open a connection in autocommit mode with foreign keys enforced.
15
+
16
+ Transactions are explicit: use ``transaction`` for multi-statement writes.
17
+ """
18
+ try:
19
+ conn = sqlite3.connect(path, isolation_level=None)
20
+ conn.row_factory = sqlite3.Row
21
+ conn.execute("PRAGMA foreign_keys = ON")
22
+ conn.execute("PRAGMA journal_mode = WAL")
23
+ except sqlite3.Error as exc:
24
+ raise StorageError(f"cannot open the project database at {path}: {exc}") from exc
25
+ return conn
26
+
27
+
28
+ @contextmanager
29
+ def transaction(conn: sqlite3.Connection) -> Iterator[sqlite3.Connection]:
30
+ """Run a block atomically: commit on success, roll back on any exception."""
31
+ conn.execute("BEGIN IMMEDIATE")
32
+ try:
33
+ yield conn
34
+ except BaseException:
35
+ conn.execute("ROLLBACK")
36
+ raise
37
+ conn.execute("COMMIT")
38
+
39
+
40
+ def open_database(paths: ProjectPaths) -> sqlite3.Connection:
41
+ """Create the state directories if needed, open the database, and migrate it."""
42
+ paths.ensure()
43
+ conn = connect(paths.database)
44
+ migrate(conn)
45
+ return conn
@@ -0,0 +1,129 @@
1
+ """Persistence for dataset versions and split assignments."""
2
+
3
+ import sqlite3
4
+ from collections.abc import Mapping
5
+ from dataclasses import dataclass
6
+
7
+ from strop.data.models import Partition
8
+ from strop.storage.database import transaction
9
+
10
+
11
+ @dataclass(frozen=True, slots=True)
12
+ class SplitRecord:
13
+ id: int
14
+ dataset_id: int
15
+ content_hash: str
16
+ skip_invalid: bool
17
+ seed: int
18
+ strategy: str
19
+ config_json: str
20
+
21
+
22
+ @dataclass(frozen=True, slots=True)
23
+ class DatasetInfo:
24
+ id: int
25
+ content_hash: str
26
+ source_path: str
27
+ row_count: int
28
+ skipped_rows: int
29
+ skip_invalid: bool
30
+
31
+
32
+ class DatasetRepository:
33
+ def __init__(self, conn: sqlite3.Connection) -> None:
34
+ self._conn = conn
35
+
36
+ def save_dataset(
37
+ self,
38
+ *,
39
+ content_hash: str,
40
+ source_path: str,
41
+ row_count: int,
42
+ skipped_rows: int,
43
+ skip_invalid: bool,
44
+ ) -> int:
45
+ """Record a dataset version (idempotent) and return its id."""
46
+ self._conn.execute(
47
+ "INSERT OR IGNORE INTO datasets "
48
+ "(content_hash, source_path, row_count, skipped_rows, skip_invalid) "
49
+ "VALUES (?, ?, ?, ?, ?)",
50
+ (content_hash, source_path, row_count, skipped_rows, int(skip_invalid)),
51
+ )
52
+ row = self._conn.execute(
53
+ "SELECT id FROM datasets WHERE content_hash = ? AND skip_invalid = ?",
54
+ (content_hash, int(skip_invalid)),
55
+ ).fetchone()
56
+ return int(row["id"])
57
+
58
+ def create_split(
59
+ self,
60
+ *,
61
+ dataset_id: int,
62
+ seed: int,
63
+ strategy: str,
64
+ config_json: str,
65
+ assignments: Mapping[str, Partition],
66
+ ) -> int:
67
+ """Store a split and all its row assignments atomically; return the split id."""
68
+ with transaction(self._conn):
69
+ cursor = self._conn.execute(
70
+ "INSERT INTO splits (dataset_id, seed, strategy, config_json) VALUES (?, ?, ?, ?)",
71
+ (dataset_id, seed, strategy, config_json),
72
+ )
73
+ split_id = int(cursor.lastrowid or 0)
74
+ self._conn.executemany(
75
+ "INSERT INTO split_assignments (split_id, row_id, partition) VALUES (?, ?, ?)",
76
+ [(split_id, row_id, part.value) for row_id, part in assignments.items()],
77
+ )
78
+ return split_id
79
+
80
+ def find_split(self, dataset_id: int, config_json: str) -> int | None:
81
+ row = self._conn.execute(
82
+ "SELECT id FROM splits WHERE dataset_id = ? AND config_json = ? ORDER BY id DESC",
83
+ (dataset_id, config_json),
84
+ ).fetchone()
85
+ return int(row["id"]) if row else None
86
+
87
+ def latest_split(self) -> SplitRecord | None:
88
+ row = self._conn.execute(
89
+ "SELECT s.id, s.dataset_id, d.content_hash, d.skip_invalid, s.seed, s.strategy, "
90
+ "s.config_json FROM splits s JOIN datasets d ON d.id = s.dataset_id "
91
+ "ORDER BY s.id DESC LIMIT 1"
92
+ ).fetchone()
93
+ if row is None:
94
+ return None
95
+ return SplitRecord(
96
+ row["id"],
97
+ row["dataset_id"],
98
+ row["content_hash"],
99
+ bool(row["skip_invalid"]),
100
+ row["seed"],
101
+ row["strategy"],
102
+ row["config_json"],
103
+ )
104
+
105
+ def assignments(self, split_id: int) -> dict[str, Partition]:
106
+ rows = self._conn.execute(
107
+ "SELECT row_id, partition FROM split_assignments WHERE split_id = ?", (split_id,)
108
+ ).fetchall()
109
+ return {r["row_id"]: Partition(r["partition"]) for r in rows}
110
+
111
+ def partition_counts(self, split_id: int) -> dict[Partition, int]:
112
+ rows = self._conn.execute(
113
+ "SELECT partition, COUNT(*) AS n FROM split_assignments WHERE split_id = ? "
114
+ "GROUP BY partition",
115
+ (split_id,),
116
+ ).fetchall()
117
+ counts = {Partition(r["partition"]): r["n"] for r in rows}
118
+ return {partition: counts.get(partition, 0) for partition in Partition}
119
+
120
+ def dataset_info(self, dataset_id: int) -> DatasetInfo:
121
+ row = self._conn.execute("SELECT * FROM datasets WHERE id = ?", (dataset_id,)).fetchone()
122
+ return DatasetInfo(
123
+ row["id"],
124
+ row["content_hash"],
125
+ row["source_path"],
126
+ row["row_count"],
127
+ row["skipped_rows"],
128
+ bool(row["skip_invalid"]),
129
+ )
@@ -0,0 +1,160 @@
1
+ """Persistence for evaluations and their per-example results."""
2
+
3
+ import json
4
+ import sqlite3
5
+ from dataclasses import asdict
6
+ from typing import Any
7
+
8
+ from strop.data.models import Partition
9
+ from strop.errors import StorageError
10
+ from strop.evaluation.models import (
11
+ EvaluationRecord,
12
+ ExampleOutcome,
13
+ NewEvaluation,
14
+ UsageSummary,
15
+ summarize_usage,
16
+ )
17
+ from strop.metrics.suite import MetricReport
18
+ from strop.storage.database import transaction
19
+
20
+ _OUTCOME_COLUMNS = (
21
+ "row_id, expected, raw_output, value, parsed_json, error, detail, finish_reason, latency_s, "
22
+ "prompt_tokens, completion_tokens, total_tokens, cost_usd, cached, scores_json, feedback"
23
+ )
24
+
25
+
26
+ _INSERT_OUTCOME = (
27
+ f"INSERT INTO example_results (evaluation_id, {_OUTCOME_COLUMNS}) "
28
+ f"VALUES ({', '.join('?' * 17)})"
29
+ )
30
+
31
+
32
+ class EvaluationRepository:
33
+ def __init__(self, conn: sqlite3.Connection) -> None:
34
+ self._conn = conn
35
+
36
+ def save(self, new: NewEvaluation) -> EvaluationRecord:
37
+ """Store an evaluation and all of its example results atomically."""
38
+ usage = summarize_usage(new.outcomes)
39
+ with transaction(self._conn):
40
+ cursor = self._conn.execute(
41
+ "INSERT INTO evaluations (prompt_number, split_id, partition, kind, run_id, "
42
+ "metrics_json, usage_json, details_json, sample_count, error_count) "
43
+ "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
44
+ (
45
+ new.prompt_number,
46
+ new.split_id,
47
+ new.partition.value,
48
+ new.kind,
49
+ new.run_id,
50
+ json.dumps(new.report.to_dict()),
51
+ json.dumps(asdict(usage)),
52
+ json.dumps(new.details),
53
+ len(new.outcomes),
54
+ sum(o.failed for o in new.outcomes),
55
+ ),
56
+ )
57
+ evaluation_id = int(cursor.lastrowid or 0)
58
+ self._conn.executemany(
59
+ _INSERT_OUTCOME, [self._row(evaluation_id, o) for o in new.outcomes]
60
+ )
61
+ return self.get(evaluation_id)
62
+
63
+ def get(self, evaluation_id: int) -> EvaluationRecord:
64
+ row = self._conn.execute(
65
+ "SELECT * FROM evaluations WHERE id = ?", (evaluation_id,)
66
+ ).fetchone()
67
+ if row is None:
68
+ raise StorageError(f"evaluation {evaluation_id} does not exist")
69
+ return _record(row)
70
+
71
+ def find(
72
+ self,
73
+ *,
74
+ prompt_number: int | None = None,
75
+ partition: Partition | None = None,
76
+ kind: str | None = None,
77
+ limit: int = 20,
78
+ ) -> list[EvaluationRecord]:
79
+ """Matching evaluations, newest first."""
80
+ clauses, args = [], []
81
+ for column, value in (
82
+ ("prompt_number", prompt_number),
83
+ ("partition", partition.value if partition else None),
84
+ ("kind", kind),
85
+ ):
86
+ if value is not None:
87
+ clauses.append(f"{column} = ?")
88
+ args.append(value)
89
+ where = f"WHERE {' AND '.join(clauses)}" if clauses else ""
90
+ rows = self._conn.execute(
91
+ f"SELECT * FROM evaluations {where} ORDER BY id DESC LIMIT ?", (*args, limit)
92
+ ).fetchall()
93
+ return [_record(row) for row in rows]
94
+
95
+ def outcomes(self, evaluation_id: int) -> list[ExampleOutcome]:
96
+ rows = self._conn.execute(
97
+ f"SELECT {_OUTCOME_COLUMNS} FROM example_results WHERE evaluation_id = ? "
98
+ "ORDER BY rowid",
99
+ (evaluation_id,),
100
+ ).fetchall()
101
+ return [_outcome(row) for row in rows]
102
+
103
+ @staticmethod
104
+ def _row(evaluation_id: int, o: ExampleOutcome) -> tuple[Any, ...]:
105
+ return (
106
+ evaluation_id,
107
+ o.row_id,
108
+ o.expected,
109
+ o.raw_output,
110
+ o.value,
111
+ json.dumps(o.parsed) if o.parsed is not None else None,
112
+ o.error,
113
+ o.detail,
114
+ o.finish_reason,
115
+ o.latency_s,
116
+ o.prompt_tokens,
117
+ o.completion_tokens,
118
+ o.total_tokens,
119
+ o.cost_usd,
120
+ int(o.cached),
121
+ json.dumps(o.scores),
122
+ o.feedback,
123
+ )
124
+
125
+
126
+ def _record(row: sqlite3.Row) -> EvaluationRecord:
127
+ return EvaluationRecord(
128
+ id=row["id"],
129
+ prompt_number=row["prompt_number"],
130
+ split_id=row["split_id"],
131
+ partition=Partition(row["partition"]),
132
+ kind=row["kind"],
133
+ report=MetricReport.from_dict(json.loads(row["metrics_json"])),
134
+ usage=UsageSummary(**json.loads(row["usage_json"])),
135
+ sample_count=row["sample_count"],
136
+ error_count=row["error_count"],
137
+ created_at=row["created_at"],
138
+ details=json.loads(row["details_json"]),
139
+ )
140
+
141
+
142
+ def _outcome(row: sqlite3.Row) -> ExampleOutcome:
143
+ return ExampleOutcome(
144
+ row_id=row["row_id"],
145
+ expected=row["expected"],
146
+ raw_output=row["raw_output"],
147
+ value=row["value"],
148
+ parsed=json.loads(row["parsed_json"]) if row["parsed_json"] else None,
149
+ error=row["error"],
150
+ detail=row["detail"],
151
+ finish_reason=row["finish_reason"],
152
+ latency_s=row["latency_s"],
153
+ prompt_tokens=row["prompt_tokens"],
154
+ completion_tokens=row["completion_tokens"],
155
+ total_tokens=row["total_tokens"],
156
+ cost_usd=row["cost_usd"],
157
+ cached=bool(row["cached"]),
158
+ scores=json.loads(row["scores_json"]),
159
+ feedback=row["feedback"],
160
+ )
@@ -0,0 +1,5 @@
1
+ -- Project-level key/value metadata (project id, creation time, tool version).
2
+ CREATE TABLE meta (
3
+ key TEXT PRIMARY KEY,
4
+ value TEXT NOT NULL
5
+ );
@@ -0,0 +1,27 @@
1
+ -- Dataset versions (identified by file content) and the splits made from them.
2
+ CREATE TABLE datasets (
3
+ id INTEGER PRIMARY KEY,
4
+ content_hash TEXT NOT NULL,
5
+ source_path TEXT NOT NULL,
6
+ row_count INTEGER NOT NULL,
7
+ skipped_rows INTEGER NOT NULL DEFAULT 0,
8
+ skip_invalid INTEGER NOT NULL DEFAULT 0,
9
+ created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
10
+ UNIQUE (content_hash, skip_invalid)
11
+ );
12
+
13
+ CREATE TABLE splits (
14
+ id INTEGER PRIMARY KEY,
15
+ dataset_id INTEGER NOT NULL REFERENCES datasets (id),
16
+ seed INTEGER NOT NULL,
17
+ strategy TEXT NOT NULL CHECK (strategy IN ('stratified', 'random')),
18
+ config_json TEXT NOT NULL,
19
+ created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP
20
+ );
21
+
22
+ CREATE TABLE split_assignments (
23
+ split_id INTEGER NOT NULL REFERENCES splits (id) ON DELETE CASCADE,
24
+ row_id TEXT NOT NULL,
25
+ partition TEXT NOT NULL CHECK (partition IN ('train', 'validation', 'test')),
26
+ PRIMARY KEY (split_id, row_id)
27
+ );
@@ -0,0 +1,22 @@
1
+ -- Immutable prompt versions. A revision never overwrites its parent.
2
+ CREATE TABLE prompt_versions (
3
+ id INTEGER PRIMARY KEY,
4
+ number INTEGER NOT NULL UNIQUE,
5
+ parent_number INTEGER REFERENCES prompt_versions (number),
6
+ content_json TEXT NOT NULL,
7
+ content_hash TEXT NOT NULL,
8
+ change_summary TEXT NOT NULL DEFAULT '',
9
+ reason TEXT NOT NULL DEFAULT '',
10
+ source TEXT NOT NULL CHECK (source IN ('user', 'optimizer')),
11
+ created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP
12
+ );
13
+
14
+ CREATE TRIGGER prompt_versions_no_update BEFORE UPDATE ON prompt_versions
15
+ BEGIN
16
+ SELECT RAISE(ABORT, 'prompt versions are immutable');
17
+ END;
18
+
19
+ CREATE TRIGGER prompt_versions_no_delete BEFORE DELETE ON prompt_versions
20
+ BEGIN
21
+ SELECT RAISE(ABORT, 'prompt versions are immutable');
22
+ END;
@@ -0,0 +1,6 @@
1
+ -- Cached model responses, keyed by a hash of everything that determines the response.
2
+ CREATE TABLE llm_cache (
3
+ key TEXT PRIMARY KEY,
4
+ response_json TEXT NOT NULL,
5
+ created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP
6
+ );