strop-cli 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- strop/__init__.py +5 -0
- strop/__main__.py +5 -0
- strop/analysis/__init__.py +17 -0
- strop/analysis/classify.py +52 -0
- strop/analysis/evidence.py +52 -0
- strop/analysis/facts.py +74 -0
- strop/analysis/patterns.py +90 -0
- strop/cli/__init__.py +5 -0
- strop/cli/analyze.py +93 -0
- strop/cli/app.py +82 -0
- strop/cli/data.py +111 -0
- strop/cli/evaluate.py +145 -0
- strop/cli/export.py +74 -0
- strop/cli/final.py +106 -0
- strop/cli/init.py +131 -0
- strop/cli/optimize.py +156 -0
- strop/cli/options.py +20 -0
- strop/cli/progress.py +18 -0
- strop/cli/prompts.py +110 -0
- strop/cli/status.py +68 -0
- strop/cli/versions.py +123 -0
- strop/config/__init__.py +16 -0
- strop/config/base.py +9 -0
- strop/config/constraints.py +33 -0
- strop/config/dataset.py +63 -0
- strop/config/enums.py +52 -0
- strop/config/io.py +52 -0
- strop/config/limits.py +19 -0
- strop/config/llm.py +42 -0
- strop/config/metrics.py +86 -0
- strop/config/optimization.py +23 -0
- strop/config/output.py +31 -0
- strop/config/overrides.py +22 -0
- strop/config/root.py +36 -0
- strop/config/rules.py +67 -0
- strop/config/split.py +23 -0
- strop/data/__init__.py +58 -0
- strop/data/loaders.py +85 -0
- strop/data/models.py +44 -0
- strop/data/rows.py +99 -0
- strop/data/splitting.py +103 -0
- strop/env.py +44 -0
- strop/errors.py +59 -0
- strop/evaluation/__init__.py +1 -0
- strop/evaluation/budget.py +67 -0
- strop/evaluation/budgeted.py +23 -0
- strop/evaluation/examples.py +49 -0
- strop/evaluation/models.py +105 -0
- strop/evaluation/parsing.py +96 -0
- strop/evaluation/runner.py +107 -0
- strop/formatting.py +14 -0
- strop/llm/__init__.py +7 -0
- strop/llm/caching.py +28 -0
- strop/llm/client.py +15 -0
- strop/llm/factory.py +14 -0
- strop/llm/litellm_client.py +90 -0
- strop/llm/mock.py +43 -0
- strop/llm/retry.py +44 -0
- strop/llm/structured.py +52 -0
- strop/llm/types.py +59 -0
- strop/log.py +49 -0
- strop/metrics/__init__.py +1 -0
- strop/metrics/basic.py +30 -0
- strop/metrics/classification.py +164 -0
- strop/metrics/judge.py +130 -0
- strop/metrics/python_metric.py +101 -0
- strop/metrics/python_runner.py +65 -0
- strop/metrics/registry.py +115 -0
- strop/metrics/rules.py +55 -0
- strop/metrics/structured.py +109 -0
- strop/metrics/suite.py +109 -0
- strop/metrics/text.py +55 -0
- strop/metrics/types.py +106 -0
- strop/optimization/__init__.py +25 -0
- strop/optimization/acceptance.py +170 -0
- strop/optimization/generator.py +92 -0
- strop/optimization/meta_prompt.py +101 -0
- strop/optimization/types.py +69 -0
- strop/prompts/__init__.py +18 -0
- strop/prompts/diff.py +139 -0
- strop/prompts/files.py +92 -0
- strop/prompts/models.py +51 -0
- strop/prompts/render.py +16 -0
- strop/prompts/template.py +45 -0
- strop/prompts/validation.py +31 -0
- strop/prompts/versions.py +34 -0
- strop/py.typed +0 -0
- strop/reporting/__init__.py +6 -0
- strop/reporting/console.py +20 -0
- strop/reporting/diff_view.py +102 -0
- strop/reporting/evaluation_view.py +141 -0
- strop/reporting/final_view.py +109 -0
- strop/reporting/prompt_export.py +58 -0
- strop/reporting/report_markdown.py +157 -0
- strop/reporting/run_view.py +118 -0
- strop/services/__init__.py +1 -0
- strop/services/analysis.py +124 -0
- strop/services/clean.py +38 -0
- strop/services/compare.py +60 -0
- strop/services/context.py +70 -0
- strop/services/data.py +65 -0
- strop/services/evaluate.py +105 -0
- strop/services/evaluator.py +169 -0
- strop/services/export.py +90 -0
- strop/services/final_test.py +69 -0
- strop/services/metrics.py +89 -0
- strop/services/optimize.py +187 -0
- strop/services/partitions.py +46 -0
- strop/services/project.py +55 -0
- strop/services/prompts.py +76 -0
- strop/services/report.py +194 -0
- strop/services/review.py +43 -0
- strop/services/scaffold.py +79 -0
- strop/services/turn.py +234 -0
- strop/storage/__init__.py +6 -0
- strop/storage/cache.py +45 -0
- strop/storage/database.py +45 -0
- strop/storage/datasets.py +129 -0
- strop/storage/evaluations.py +160 -0
- strop/storage/migrations/001_initial.sql +5 -0
- strop/storage/migrations/002_datasets.sql +27 -0
- strop/storage/migrations/003_prompt_versions.sql +22 -0
- strop/storage/migrations/004_llm_cache.sql +6 -0
- strop/storage/migrations/005_evaluations.sql +37 -0
- strop/storage/migrations/006_runs.sql +52 -0
- strop/storage/migrations/007_example_feedback.sql +2 -0
- strop/storage/migrations/__init__.py +5 -0
- strop/storage/migrator.py +91 -0
- strop/storage/paths.py +30 -0
- strop/storage/prompts.py +96 -0
- strop/storage/run_models.py +75 -0
- strop/storage/runs.py +263 -0
- strop_cli-0.1.0.dist-info/METADATA +242 -0
- strop_cli-0.1.0.dist-info/RECORD +137 -0
- strop_cli-0.1.0.dist-info/WHEEL +4 -0
- strop_cli-0.1.0.dist-info/entry_points.txt +2 -0
- strop_cli-0.1.0.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,187 @@
|
|
|
1
|
+
"""The optimization loop: run turns until a stop condition is reached."""
|
|
2
|
+
|
|
3
|
+
import asyncio
|
|
4
|
+
import sqlite3
|
|
5
|
+
from dataclasses import dataclass, field
|
|
6
|
+
|
|
7
|
+
from strop.config.limits import BudgetsConfig
|
|
8
|
+
from strop.config.optimization import OptimizationConfig
|
|
9
|
+
from strop.data.models import Partition
|
|
10
|
+
from strop.errors import BudgetExceededError, ConfigError, DataError
|
|
11
|
+
from strop.evaluation.models import EvaluationRecord
|
|
12
|
+
from strop.metrics.types import Direction
|
|
13
|
+
from strop.prompts.versions import PromptVersion
|
|
14
|
+
from strop.services.context import LoopContext, Say, build_context
|
|
15
|
+
from strop.services.project import Project
|
|
16
|
+
from strop.services.prompts import ensure_initial_version
|
|
17
|
+
from strop.services.turn import TurnRunner
|
|
18
|
+
from strop.storage.runs import CandidateRecord, RunRecord, TurnRecord
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
@dataclass(frozen=True)
|
|
22
|
+
class RunSummary:
|
|
23
|
+
"""What an optimization run did and why it stopped."""
|
|
24
|
+
|
|
25
|
+
run: RunRecord
|
|
26
|
+
initial: PromptVersion
|
|
27
|
+
start: PromptVersion
|
|
28
|
+
final: PromptVersion
|
|
29
|
+
baseline: EvaluationRecord | None
|
|
30
|
+
result: EvaluationRecord | None
|
|
31
|
+
turns: list[TurnRecord]
|
|
32
|
+
candidates: list[CandidateRecord]
|
|
33
|
+
stop_reason: str
|
|
34
|
+
stop_detail: str
|
|
35
|
+
pending: CandidateRecord | None
|
|
36
|
+
calls: int
|
|
37
|
+
cost_usd: float | None
|
|
38
|
+
warnings: list[str] = field(default_factory=list)
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
async def optimize(
|
|
42
|
+
project: Project,
|
|
43
|
+
conn: sqlite3.Connection,
|
|
44
|
+
settings: OptimizationConfig,
|
|
45
|
+
budgets: BudgetsConfig,
|
|
46
|
+
*,
|
|
47
|
+
resume: bool,
|
|
48
|
+
use_cache: bool,
|
|
49
|
+
say: Say,
|
|
50
|
+
) -> RunSummary:
|
|
51
|
+
"""Run (or resume) the loop: analyze failures, propose revisions, test, and decide."""
|
|
52
|
+
ctx = build_context(project, conn, settings, budgets, use_cache=use_cache, say=say)
|
|
53
|
+
validation = len(ctx.evaluator.rows(Partition.VALIDATION))
|
|
54
|
+
if validation < settings.min_validation_samples:
|
|
55
|
+
raise DataError(
|
|
56
|
+
f"the validation partition has {validation} rows, fewer than the "
|
|
57
|
+
f"{settings.min_validation_samples} needed to compare prompts; add data"
|
|
58
|
+
)
|
|
59
|
+
run = _begin(ctx, resume)
|
|
60
|
+
runner = TurnRunner(ctx, run)
|
|
61
|
+
reason, detail = "", ""
|
|
62
|
+
try:
|
|
63
|
+
reason = await _turns(ctx, run, runner)
|
|
64
|
+
except BudgetExceededError as exc:
|
|
65
|
+
reason, detail = "budget", str(exc)
|
|
66
|
+
except (KeyboardInterrupt, asyncio.CancelledError):
|
|
67
|
+
_save(ctx, run.id, status="interrupted")
|
|
68
|
+
raise
|
|
69
|
+
except BaseException as exc:
|
|
70
|
+
_save(ctx, run.id, status="failed", stop_reason=str(exc)[:500])
|
|
71
|
+
raise
|
|
72
|
+
status = {"awaiting_approval": "paused", "budget": "interrupted"}.get(reason, "completed")
|
|
73
|
+
_save(ctx, run.id, status=status, stop_reason=reason, finished=status == "completed")
|
|
74
|
+
return _summary(ctx, run.id, reason, detail)
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def _begin(ctx: LoopContext, resume: bool) -> RunRecord:
|
|
78
|
+
pending = ctx.runs.pending()
|
|
79
|
+
if pending:
|
|
80
|
+
ref = f"v{pending[0].version_number}"
|
|
81
|
+
raise ConfigError(
|
|
82
|
+
f"{ref} is waiting for a decision; run 'strop approve {ref}' or "
|
|
83
|
+
f"'strop reject {ref}' first"
|
|
84
|
+
)
|
|
85
|
+
if resume:
|
|
86
|
+
run = ctx.runs.resumable_run()
|
|
87
|
+
if run is None:
|
|
88
|
+
raise ConfigError(
|
|
89
|
+
"there is no unfinished run to resume; start one with 'strop optimize'"
|
|
90
|
+
)
|
|
91
|
+
ctx.runs.update_run(run.id, status="running")
|
|
92
|
+
return ctx.runs.get_run(run.id)
|
|
93
|
+
unfinished = ctx.runs.resumable_run()
|
|
94
|
+
if unfinished is not None:
|
|
95
|
+
ctx.runs.update_run(unfinished.id, status="interrupted")
|
|
96
|
+
best = ctx.prompts.best() or ensure_initial_version(ctx.project, ctx.prompts)
|
|
97
|
+
return ctx.runs.create_run(best.number, ctx.settings.model_dump(mode="json"))
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
async def _turns(ctx: LoopContext, run: RunRecord, runner: TurnRunner) -> str:
|
|
101
|
+
stale = run.stale_turns
|
|
102
|
+
while True:
|
|
103
|
+
turns = ctx.runs.turns(run.id)
|
|
104
|
+
if sum(t.status == "done" for t in turns) >= ctx.settings.max_turns:
|
|
105
|
+
return "max_turns"
|
|
106
|
+
best = _best(ctx)
|
|
107
|
+
record = await runner.validation_record(best)
|
|
108
|
+
if _target_reached(ctx, record):
|
|
109
|
+
return "target_reached"
|
|
110
|
+
open_turn = next((t for t in turns if t.status != "done"), None)
|
|
111
|
+
number = open_turn.number if open_turn else (turns[-1].number + 1 if turns else 1)
|
|
112
|
+
result = await runner.run(number, best)
|
|
113
|
+
if result.awaiting is not None:
|
|
114
|
+
return "awaiting_approval"
|
|
115
|
+
stale = 0 if result.improved else stale + 1
|
|
116
|
+
_save(ctx, run.id, stale_turns=stale)
|
|
117
|
+
if result.no_failures:
|
|
118
|
+
return "no_failures"
|
|
119
|
+
if stale >= ctx.settings.patience:
|
|
120
|
+
return "patience"
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
def _best(ctx: LoopContext) -> PromptVersion:
|
|
124
|
+
best = ctx.prompts.best()
|
|
125
|
+
if best is None:
|
|
126
|
+
raise ConfigError("no best prompt is set; run 'strop baseline' first")
|
|
127
|
+
return best
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
def _target_reached(ctx: LoopContext, record: EvaluationRecord) -> bool:
|
|
131
|
+
target = ctx.settings.target
|
|
132
|
+
value = record.report.primary_value.value
|
|
133
|
+
if target is None or value is None:
|
|
134
|
+
return False
|
|
135
|
+
if ctx.evaluator.suite.direction(record.report.primary) is Direction.MAXIMIZE:
|
|
136
|
+
return value >= target
|
|
137
|
+
return value <= target
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
def _cost(ctx: LoopContext) -> float | None:
|
|
141
|
+
"""The known cost so far, or None when no call reported one."""
|
|
142
|
+
budget = ctx.budget
|
|
143
|
+
return budget.cost_usd if budget.calls > budget.cost_unknown_calls else None
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
def _save(
|
|
147
|
+
ctx: LoopContext,
|
|
148
|
+
run_id: int,
|
|
149
|
+
*,
|
|
150
|
+
status: str | None = None,
|
|
151
|
+
stale_turns: int | None = None,
|
|
152
|
+
stop_reason: str | None = None,
|
|
153
|
+
finished: bool = False,
|
|
154
|
+
) -> None:
|
|
155
|
+
"""Update a run and record the calls and cost used so far."""
|
|
156
|
+
ctx.runs.update_run(
|
|
157
|
+
run_id,
|
|
158
|
+
status=status,
|
|
159
|
+
stale_turns=stale_turns,
|
|
160
|
+
stop_reason=stop_reason,
|
|
161
|
+
model_calls=ctx.budget.calls,
|
|
162
|
+
cost_usd=_cost(ctx),
|
|
163
|
+
finished=finished,
|
|
164
|
+
)
|
|
165
|
+
|
|
166
|
+
|
|
167
|
+
def _summary(ctx: LoopContext, run_id: int, reason: str, detail: str) -> RunSummary:
|
|
168
|
+
run = ctx.runs.get_run(run_id)
|
|
169
|
+
start = ctx.prompts.get(run.start_prompt)
|
|
170
|
+
final = _best(ctx)
|
|
171
|
+
pending = next(iter(ctx.runs.pending()), None)
|
|
172
|
+
return RunSummary(
|
|
173
|
+
run=run,
|
|
174
|
+
initial=ctx.prompts.first() or start,
|
|
175
|
+
start=start,
|
|
176
|
+
final=final,
|
|
177
|
+
baseline=ctx.evaluator.reusable(start, Partition.VALIDATION),
|
|
178
|
+
result=ctx.evaluator.reusable(final, Partition.VALIDATION),
|
|
179
|
+
turns=ctx.runs.turns(run_id),
|
|
180
|
+
candidates=ctx.runs.candidates(run_id),
|
|
181
|
+
stop_reason=reason,
|
|
182
|
+
stop_detail=detail,
|
|
183
|
+
pending=pending,
|
|
184
|
+
calls=ctx.budget.calls,
|
|
185
|
+
cost_usd=_cost(ctx),
|
|
186
|
+
warnings=ctx.budget.warnings(),
|
|
187
|
+
)
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
"""Loading the dataset rows of the active split, grouped by partition."""
|
|
2
|
+
|
|
3
|
+
from dataclasses import dataclass, field
|
|
4
|
+
|
|
5
|
+
from strop.data import load_dataset, require_valid
|
|
6
|
+
from strop.data.models import Partition, Row
|
|
7
|
+
from strop.errors import ConfigError, DataError
|
|
8
|
+
from strop.services.project import Project
|
|
9
|
+
from strop.storage.datasets import DatasetRepository
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
@dataclass(frozen=True)
|
|
13
|
+
class PartitionedData:
|
|
14
|
+
"""Rows of the latest split. The test partition is only handed out on explicit request."""
|
|
15
|
+
|
|
16
|
+
split_id: int
|
|
17
|
+
seed: int
|
|
18
|
+
rows: dict[Partition, list[Row]] = field(default_factory=dict)
|
|
19
|
+
|
|
20
|
+
def get(self, partition: Partition) -> list[Row]:
|
|
21
|
+
return self.rows.get(partition, [])
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def load_partitions(project: Project, repo: DatasetRepository) -> PartitionedData:
|
|
25
|
+
"""Read the dataset and attach the stored split assignments.
|
|
26
|
+
|
|
27
|
+
Fails if there is no split or if the dataset file changed since the split was made.
|
|
28
|
+
"""
|
|
29
|
+
split = repo.latest_split()
|
|
30
|
+
if split is None:
|
|
31
|
+
raise ConfigError("no data split yet; run 'strop data split' first")
|
|
32
|
+
dataset = require_valid(
|
|
33
|
+
load_dataset(project.config, project.root), skip_invalid=split.skip_invalid
|
|
34
|
+
)
|
|
35
|
+
if dataset.content_hash != split.content_hash:
|
|
36
|
+
raise DataError(
|
|
37
|
+
"the dataset file changed since the split was created; run 'strop data split' again"
|
|
38
|
+
)
|
|
39
|
+
assignments = repo.assignments(split.id)
|
|
40
|
+
rows: dict[Partition, list[Row]] = {p: [] for p in Partition}
|
|
41
|
+
for row in dataset.rows:
|
|
42
|
+
partition = assignments.get(row.id)
|
|
43
|
+
if partition is None:
|
|
44
|
+
raise DataError(f"row '{row.id}' has no partition; run 'strop data split' again")
|
|
45
|
+
rows[partition].append(row)
|
|
46
|
+
return PartitionedData(split.id, split.seed, rows)
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
"""An opened Strop project: its directory, validated configuration, and database."""
|
|
2
|
+
|
|
3
|
+
import sqlite3
|
|
4
|
+
from collections.abc import Iterator
|
|
5
|
+
from contextlib import contextmanager
|
|
6
|
+
from dataclasses import dataclass, replace
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
|
|
9
|
+
from strop.config import StropConfig, load_config
|
|
10
|
+
from strop.config.llm import ModelSpec
|
|
11
|
+
from strop.env import load_env_file
|
|
12
|
+
from strop.storage import ProjectPaths, open_database
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
@dataclass(frozen=True)
|
|
16
|
+
class Project:
|
|
17
|
+
root: Path
|
|
18
|
+
paths: ProjectPaths
|
|
19
|
+
config: StropConfig
|
|
20
|
+
|
|
21
|
+
def connect(self) -> sqlite3.Connection:
|
|
22
|
+
"""Open the project database, creating and migrating it if needed."""
|
|
23
|
+
return open_database(self.paths)
|
|
24
|
+
|
|
25
|
+
def with_models(self, overrides: dict[str, str | None]) -> "Project":
|
|
26
|
+
"""A copy that uses the given model names for some roles; None values are ignored."""
|
|
27
|
+
models = self.config.models
|
|
28
|
+
changes: dict[str, ModelSpec] = {}
|
|
29
|
+
for role, name in overrides.items():
|
|
30
|
+
if name:
|
|
31
|
+
current: ModelSpec | None = getattr(models, role)
|
|
32
|
+
changes[role] = (
|
|
33
|
+
current.model_copy(update={"model": name}) if current else ModelSpec(model=name)
|
|
34
|
+
)
|
|
35
|
+
if not changes:
|
|
36
|
+
return self
|
|
37
|
+
config = self.config.model_copy(update={"models": models.model_copy(update=changes)})
|
|
38
|
+
return replace(self, config=config)
|
|
39
|
+
|
|
40
|
+
@contextmanager
|
|
41
|
+
def database(self) -> Iterator[sqlite3.Connection]:
|
|
42
|
+
"""Open the project database for the duration of a ``with`` block."""
|
|
43
|
+
conn = self.connect()
|
|
44
|
+
try:
|
|
45
|
+
yield conn
|
|
46
|
+
finally:
|
|
47
|
+
conn.close()
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def open_project(root: Path) -> Project:
|
|
51
|
+
"""Load strop.yaml from ``root``; raise ``ConfigError`` if it is missing or invalid."""
|
|
52
|
+
paths = ProjectPaths(root)
|
|
53
|
+
config = load_config(paths.config_file)
|
|
54
|
+
load_env_file(root / ".env")
|
|
55
|
+
return Project(root, paths, config)
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
"""Prompt version workflows."""
|
|
2
|
+
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
|
|
5
|
+
from strop.errors import ConfigError
|
|
6
|
+
from strop.prompts import PromptTemplate, check_variables, load_prompt_file
|
|
7
|
+
from strop.prompts.versions import PromptVersion, parse_version_ref
|
|
8
|
+
from strop.services.project import Project
|
|
9
|
+
from strop.storage.prompts import PromptRepository
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def load_project_prompt(project: Project, path: Path | None = None) -> PromptTemplate:
|
|
13
|
+
"""Parse a prompt file and check it only uses variables the dataset provides."""
|
|
14
|
+
template = load_prompt_file(path or project.root / project.config.prompt.file)
|
|
15
|
+
check_variables(template.variables, project.config.dataset.input_columns)
|
|
16
|
+
return template
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def ensure_initial_version(project: Project, repo: PromptRepository) -> PromptVersion:
|
|
20
|
+
"""Register the prompt file as v1 on first use, and make it the best prompt."""
|
|
21
|
+
existing = repo.first()
|
|
22
|
+
if existing is not None:
|
|
23
|
+
return existing
|
|
24
|
+
template = load_project_prompt(project)
|
|
25
|
+
version = repo.add(
|
|
26
|
+
template,
|
|
27
|
+
parent_number=None,
|
|
28
|
+
change_summary="initial prompt",
|
|
29
|
+
reason=f"registered from {project.config.prompt.file}",
|
|
30
|
+
)
|
|
31
|
+
repo.set_best(version.number)
|
|
32
|
+
return version
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def has_unregistered_edits(project: Project, repo: PromptRepository) -> bool:
|
|
36
|
+
"""True if the prompt file differs from every stored version."""
|
|
37
|
+
template = load_prompt_file(project.root / project.config.prompt.file)
|
|
38
|
+
return repo.find_by_hash(template.content_hash) is None
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def add_version_from_file(
|
|
42
|
+
project: Project,
|
|
43
|
+
repo: PromptRepository,
|
|
44
|
+
file: Path,
|
|
45
|
+
*,
|
|
46
|
+
parent: str | None,
|
|
47
|
+
reason: str,
|
|
48
|
+
use: bool,
|
|
49
|
+
) -> PromptVersion:
|
|
50
|
+
"""Save a manual edit as a new immutable version, optionally making it the best prompt."""
|
|
51
|
+
ensure_initial_version(project, repo)
|
|
52
|
+
template = load_project_prompt(project, file)
|
|
53
|
+
duplicate = repo.find_by_hash(template.content_hash)
|
|
54
|
+
if duplicate is not None:
|
|
55
|
+
raise ConfigError(f"this prompt is identical to the existing version {duplicate.ref}")
|
|
56
|
+
parent_number = resolve_version(repo, parent).number
|
|
57
|
+
version = repo.add(
|
|
58
|
+
template,
|
|
59
|
+
parent_number=parent_number,
|
|
60
|
+
change_summary="manual edit",
|
|
61
|
+
reason=reason,
|
|
62
|
+
source="user",
|
|
63
|
+
)
|
|
64
|
+
if use:
|
|
65
|
+
repo.set_best(version.number)
|
|
66
|
+
return version
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def resolve_version(repo: PromptRepository, ref: str | None) -> PromptVersion:
|
|
70
|
+
"""The version named by ``ref`` (like ``v3``), or the best version when ``ref`` is None."""
|
|
71
|
+
if ref is not None:
|
|
72
|
+
return repo.get(parse_version_ref(ref))
|
|
73
|
+
best = repo.best()
|
|
74
|
+
if best is None:
|
|
75
|
+
raise ConfigError("no prompt versions yet; run 'strop baseline' or 'strop prompt add'")
|
|
76
|
+
return best
|
strop/services/report.py
ADDED
|
@@ -0,0 +1,194 @@
|
|
|
1
|
+
"""Collecting everything a final report needs from the project database."""
|
|
2
|
+
|
|
3
|
+
import sqlite3
|
|
4
|
+
from datetime import UTC, datetime
|
|
5
|
+
from typing import Any
|
|
6
|
+
|
|
7
|
+
from strop import __version__
|
|
8
|
+
from strop.analysis import build_facts
|
|
9
|
+
from strop.data.models import Partition
|
|
10
|
+
from strop.evaluation.models import EvaluationRecord
|
|
11
|
+
from strop.prompts.diff import diff_templates
|
|
12
|
+
from strop.prompts.versions import PromptVersion
|
|
13
|
+
from strop.reporting.evaluation_view import record_payload
|
|
14
|
+
from strop.services.project import Project
|
|
15
|
+
from strop.storage.datasets import DatasetRepository
|
|
16
|
+
from strop.storage.evaluations import EvaluationRepository
|
|
17
|
+
from strop.storage.prompts import PromptRepository
|
|
18
|
+
from strop.storage.runs import RunRecord, RunRepository
|
|
19
|
+
|
|
20
|
+
SMALL_VALIDATION = 30
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def build_report(
|
|
24
|
+
project: Project, conn: sqlite3.Connection, selected: PromptVersion
|
|
25
|
+
) -> dict[str, Any]:
|
|
26
|
+
"""A machine-readable report for ``selected``; missing data is marked unavailable."""
|
|
27
|
+
prompts, evals = PromptRepository(conn), EvaluationRepository(conn)
|
|
28
|
+
datasets, runs = DatasetRepository(conn), RunRepository(conn)
|
|
29
|
+
initial = prompts.first() or selected
|
|
30
|
+
split = datasets.latest_split()
|
|
31
|
+
counts = datasets.partition_counts(split.id) if split else {}
|
|
32
|
+
results = {
|
|
33
|
+
"initial_validation": _latest(evals, initial, Partition.VALIDATION),
|
|
34
|
+
"selected_validation": _latest(evals, selected, Partition.VALIDATION),
|
|
35
|
+
"initial_test": _latest(evals, initial, Partition.TEST),
|
|
36
|
+
"selected_test": _latest(evals, selected, Partition.TEST),
|
|
37
|
+
}
|
|
38
|
+
payload: dict[str, Any] = {
|
|
39
|
+
"generated_at": datetime.now(UTC).isoformat(timespec="seconds"),
|
|
40
|
+
"strop_version": __version__,
|
|
41
|
+
"project": {
|
|
42
|
+
"name": project.config.project.name,
|
|
43
|
+
"task": project.config.project.task_description,
|
|
44
|
+
},
|
|
45
|
+
"dataset": _dataset(datasets, split.dataset_id) if split else None,
|
|
46
|
+
"split": (
|
|
47
|
+
{"id": split.id, "seed": split.seed, "counts": {p.value: n for p, n in counts.items()}}
|
|
48
|
+
if split
|
|
49
|
+
else None
|
|
50
|
+
),
|
|
51
|
+
"models": _models(project),
|
|
52
|
+
"metrics": _metrics(project),
|
|
53
|
+
"prompt": {
|
|
54
|
+
"selected": selected.number,
|
|
55
|
+
"initial": initial.number,
|
|
56
|
+
"best": prompts.best_number(),
|
|
57
|
+
},
|
|
58
|
+
"results": {
|
|
59
|
+
key: record_payload(rec, f"v{rec.prompt_number}") if rec else None
|
|
60
|
+
for key, rec in results.items()
|
|
61
|
+
},
|
|
62
|
+
"per_class": _per_class(results["selected_validation"]),
|
|
63
|
+
"versions": [_version(v, runs.for_version(v.number)) for v in prompts.list()],
|
|
64
|
+
"runs": [_run(r, runs) for r in runs.all_runs()],
|
|
65
|
+
"diff": diff_templates(initial.template, selected.template).to_payload(),
|
|
66
|
+
"error_analysis": _error_analysis(project, evals, selected),
|
|
67
|
+
"usage": _usage(evals),
|
|
68
|
+
}
|
|
69
|
+
payload["limitations"] = limitations(payload)
|
|
70
|
+
return payload
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def limitations(report: dict[str, Any]) -> list[str]:
|
|
74
|
+
"""Plain statements about what this report cannot show."""
|
|
75
|
+
notes = []
|
|
76
|
+
validation = (report["results"].get("selected_validation") or {}).get("samples")
|
|
77
|
+
if validation is not None and validation < SMALL_VALIDATION:
|
|
78
|
+
notes.append(
|
|
79
|
+
f"The validation partition has only {validation} examples, so small score "
|
|
80
|
+
"differences may be noise."
|
|
81
|
+
)
|
|
82
|
+
if (report["split"] or {}).get("counts", {}).get("test", 0) == 0:
|
|
83
|
+
notes.append("There is no test partition, so there is no held-out result.")
|
|
84
|
+
elif report["results"].get("selected_test") is None:
|
|
85
|
+
notes.append("The selected prompt has not been evaluated on the test partition yet.")
|
|
86
|
+
if any(m["subjective"] for r in report["results"].values() if r for m in r["metrics"].values()):
|
|
87
|
+
notes.append("LLM-judged metrics are subjective estimates, not ground truth.")
|
|
88
|
+
if report["usage"]["cost_usd"] is None:
|
|
89
|
+
notes.append("Cost is unavailable: no provider call reported a price.")
|
|
90
|
+
notes.append("Results come from one dataset split and depend on the models that were used.")
|
|
91
|
+
return notes
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def _latest(
|
|
95
|
+
evals: EvaluationRepository, version: PromptVersion, partition: Partition
|
|
96
|
+
) -> EvaluationRecord | None:
|
|
97
|
+
found = evals.find(prompt_number=version.number, partition=partition, limit=1)
|
|
98
|
+
return found[0] if found else None
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def _dataset(datasets: DatasetRepository, dataset_id: int) -> dict[str, Any]:
|
|
102
|
+
info = datasets.dataset_info(dataset_id)
|
|
103
|
+
return {
|
|
104
|
+
"path": info.source_path,
|
|
105
|
+
"content_hash": info.content_hash,
|
|
106
|
+
"rows": info.row_count,
|
|
107
|
+
"skipped_rows": info.skipped_rows,
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
def _models(project: Project) -> dict[str, str | None]:
|
|
112
|
+
models = project.config.models
|
|
113
|
+
return {
|
|
114
|
+
role: (spec.model if spec else None)
|
|
115
|
+
for role, spec in (
|
|
116
|
+
("target", models.target),
|
|
117
|
+
("optimizer", models.optimizer),
|
|
118
|
+
("analyzer", models.analyzer),
|
|
119
|
+
("judge", models.judge),
|
|
120
|
+
)
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
def _metrics(project: Project) -> dict[str, Any]:
|
|
125
|
+
config = project.config.metrics
|
|
126
|
+
return {
|
|
127
|
+
"primary": config.primary,
|
|
128
|
+
"secondary": list(config.secondary),
|
|
129
|
+
"constraints": {k: b.model_dump(exclude_none=True) for k, b in config.constraints.items()},
|
|
130
|
+
"rules": sorted(config.rules),
|
|
131
|
+
"judges": sorted(config.judges),
|
|
132
|
+
"custom": config.python is not None,
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
def _per_class(record: EvaluationRecord | None) -> dict[str, Any]:
|
|
137
|
+
if record is None:
|
|
138
|
+
return {}
|
|
139
|
+
per_class: dict[str, Any] = record.report.primary_value.details.get("per_class", {})
|
|
140
|
+
return per_class
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
def _version(version: PromptVersion, decision: Any) -> dict[str, Any]:
|
|
144
|
+
return {
|
|
145
|
+
"version": version.number,
|
|
146
|
+
"parent": version.parent_number,
|
|
147
|
+
"source": version.source,
|
|
148
|
+
"change_summary": version.change_summary,
|
|
149
|
+
"status": decision.status if decision else None,
|
|
150
|
+
"primary_delta": decision.primary_delta if decision else None,
|
|
151
|
+
"decision_reason": decision.reason if decision else None,
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
def _run(run: RunRecord, runs: RunRepository) -> dict[str, Any]:
|
|
156
|
+
return {
|
|
157
|
+
"id": run.id,
|
|
158
|
+
"status": run.status,
|
|
159
|
+
"stop_reason": run.stop_reason,
|
|
160
|
+
"turns": len([t for t in runs.turns(run.id) if t.status == "done"]),
|
|
161
|
+
"model_calls": run.model_calls,
|
|
162
|
+
"cost_usd": run.cost_usd,
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
def _error_analysis(
|
|
167
|
+
project: Project, evals: EvaluationRepository, selected: PromptVersion
|
|
168
|
+
) -> dict[str, Any] | None:
|
|
169
|
+
"""Error counts from the latest validation outcomes; counts only, no example content."""
|
|
170
|
+
record = _latest(evals, selected, Partition.VALIDATION)
|
|
171
|
+
if record is None:
|
|
172
|
+
return None
|
|
173
|
+
outcomes = evals.outcomes(record.id)
|
|
174
|
+
facts = build_facts(outcomes, [], project.config.output.type, record.report.primary)
|
|
175
|
+
return {
|
|
176
|
+
"evaluation": record.id,
|
|
177
|
+
"examples": facts.total,
|
|
178
|
+
"failed": facts.failed,
|
|
179
|
+
"codes": dict(facts.codes),
|
|
180
|
+
"confusions": [{"expected": e, "got": g, "count": n} for e, g, n in facts.confusions],
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
|
|
184
|
+
def _usage(evals: EvaluationRepository) -> dict[str, Any]:
|
|
185
|
+
records = evals.find(limit=100000)
|
|
186
|
+
calls = sum(r.usage.calls - r.usage.cached_calls for r in records)
|
|
187
|
+
costs = [r.usage.cost_usd for r in records if r.usage.cost_usd is not None]
|
|
188
|
+
tokens = [r.usage.total_tokens for r in records if r.usage.total_tokens is not None]
|
|
189
|
+
return {
|
|
190
|
+
"evaluations": len(records),
|
|
191
|
+
"model_calls": calls,
|
|
192
|
+
"total_tokens": sum(tokens) if tokens else None,
|
|
193
|
+
"cost_usd": sum(costs) if costs else None,
|
|
194
|
+
}
|
strop/services/review.py
ADDED
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
"""Human decisions on candidates that are waiting for approval."""
|
|
2
|
+
|
|
3
|
+
import sqlite3
|
|
4
|
+
|
|
5
|
+
from strop.errors import ConfigError
|
|
6
|
+
from strop.prompts.versions import parse_version_ref
|
|
7
|
+
from strop.storage.prompts import PromptRepository
|
|
8
|
+
from strop.storage.runs import CandidateRecord, RunRepository
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def _pending(runs: RunRepository, ref: str) -> CandidateRecord:
|
|
12
|
+
number = parse_version_ref(ref)
|
|
13
|
+
candidate = runs.for_version(number)
|
|
14
|
+
if candidate is None or candidate.status != "pending":
|
|
15
|
+
raise ConfigError(f"v{number} is not waiting for a decision")
|
|
16
|
+
return candidate
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def approve(conn: sqlite3.Connection, ref: str) -> CandidateRecord:
|
|
20
|
+
"""Make a pending candidate the best prompt and close its turn."""
|
|
21
|
+
runs, prompts = RunRepository(conn), PromptRepository(conn)
|
|
22
|
+
candidate = _pending(runs, ref)
|
|
23
|
+
number = candidate.version_number
|
|
24
|
+
if number is None:
|
|
25
|
+
raise ConfigError(f"{ref} has no prompt version")
|
|
26
|
+
prompts.set_best(number)
|
|
27
|
+
runs.set_candidate_status(candidate.id, "accepted", f"{candidate.reason}; approved by the user")
|
|
28
|
+
runs.update_turn(candidate.run_id, candidate.turn, status="done", best_after=number)
|
|
29
|
+
runs.update_run(candidate.run_id, stale_turns=0)
|
|
30
|
+
return runs.get_candidate(candidate.id)
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def reject(conn: sqlite3.Connection, ref: str, reason: str = "") -> CandidateRecord:
|
|
34
|
+
"""Discard a pending candidate; the best prompt stays as it was."""
|
|
35
|
+
runs = RunRepository(conn)
|
|
36
|
+
candidate = _pending(runs, ref)
|
|
37
|
+
why = f"rejected by the user: {reason}" if reason else "rejected by the user"
|
|
38
|
+
runs.set_candidate_status(candidate.id, "rejected", why)
|
|
39
|
+
turn = runs.get_turn(candidate.run_id, candidate.turn)
|
|
40
|
+
runs.update_turn(candidate.run_id, candidate.turn, status="done", best_after=turn.best_before)
|
|
41
|
+
run = runs.get_run(candidate.run_id)
|
|
42
|
+
runs.update_run(candidate.run_id, stale_turns=run.stale_turns + 1)
|
|
43
|
+
return runs.get_candidate(candidate.id)
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
"""Creating a new Strop project on disk."""
|
|
2
|
+
|
|
3
|
+
from datetime import UTC, datetime
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
|
|
6
|
+
from strop import __version__
|
|
7
|
+
from strop.config import OutputType, StropConfig, dump_config
|
|
8
|
+
from strop.errors import ConfigError
|
|
9
|
+
from strop.evaluation.parsing import load_output_schema
|
|
10
|
+
from strop.prompts import PromptMessage, PromptTemplate, dump_prompt
|
|
11
|
+
from strop.services.metrics import validate_metric_names
|
|
12
|
+
from strop.storage import ProjectPaths, open_database
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def scaffold_project(config: StropConfig, root: Path, *, force: bool = False) -> list[str]:
|
|
16
|
+
"""Write strop.yaml, a starter prompt if none exists, and the state directory.
|
|
17
|
+
|
|
18
|
+
Returns the entries that were created or rewritten, relative to ``root``.
|
|
19
|
+
"""
|
|
20
|
+
dataset_file = root / config.dataset.path
|
|
21
|
+
if not dataset_file.is_file():
|
|
22
|
+
raise ConfigError(f"dataset file not found: {dataset_file}")
|
|
23
|
+
load_output_schema(root, config.output)
|
|
24
|
+
validate_metric_names(config)
|
|
25
|
+
if config.metrics.python is not None and not (root / config.metrics.python.file).is_file():
|
|
26
|
+
raise ConfigError(f"metric file not found: {root / config.metrics.python.file}")
|
|
27
|
+
|
|
28
|
+
paths = ProjectPaths(root)
|
|
29
|
+
dump_config(config, paths.config_file, force=force)
|
|
30
|
+
created = [paths.config_file.name]
|
|
31
|
+
|
|
32
|
+
prompt_file = root / config.prompt.file
|
|
33
|
+
if not prompt_file.exists():
|
|
34
|
+
prompt_file.parent.mkdir(parents=True, exist_ok=True)
|
|
35
|
+
prompt_file.write_text(starter_prompt(config), encoding="utf-8")
|
|
36
|
+
created.append(str(config.prompt.file))
|
|
37
|
+
|
|
38
|
+
conn = open_database(paths)
|
|
39
|
+
try:
|
|
40
|
+
conn.executemany(
|
|
41
|
+
"INSERT OR REPLACE INTO meta (key, value) VALUES (?, ?)",
|
|
42
|
+
[
|
|
43
|
+
("project_name", config.project.name),
|
|
44
|
+
("strop_version", __version__),
|
|
45
|
+
("created_at", datetime.now(UTC).isoformat()),
|
|
46
|
+
],
|
|
47
|
+
)
|
|
48
|
+
finally:
|
|
49
|
+
conn.close()
|
|
50
|
+
created.append(f"{paths.state_dir.name}/")
|
|
51
|
+
return created
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def data_recipients(config: StropConfig) -> list[str]:
|
|
55
|
+
"""Provider names that will receive dataset content, from the configured models."""
|
|
56
|
+
models = config.models
|
|
57
|
+
specs = (models.target, models.optimizer, models.analyzer, models.judge)
|
|
58
|
+
return sorted({spec.model.split("/", 1)[0] for spec in specs if spec is not None})
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def starter_prompt(config: StropConfig) -> str:
|
|
62
|
+
"""A minimal prompt file the user is expected to edit before running.
|
|
63
|
+
|
|
64
|
+
The format follows the prompt file extension: YAML messages for ``.yaml``/``.yml``,
|
|
65
|
+
a single plain-text template otherwise.
|
|
66
|
+
"""
|
|
67
|
+
system = config.project.task_description.strip()
|
|
68
|
+
if config.output.type is OutputType.LABEL:
|
|
69
|
+
labels = ", ".join(config.output.labels)
|
|
70
|
+
system += f"\nRespond with exactly one of these labels and nothing else: {labels}."
|
|
71
|
+
elif config.output.type is OutputType.JSON:
|
|
72
|
+
system += "\nRespond with a single valid JSON object and nothing else."
|
|
73
|
+
user = "\n".join(f"{name}: {{{{{name}}}}}" for name in config.dataset.input_columns)
|
|
74
|
+
if config.prompt.file.suffix.lower() not in {".yaml", ".yml"}:
|
|
75
|
+
return f"{system}\n\n{user}\n"
|
|
76
|
+
template = PromptTemplate(
|
|
77
|
+
(PromptMessage("system", system), PromptMessage("user", user)), frozenset({"system"})
|
|
78
|
+
)
|
|
79
|
+
return dump_prompt(template)
|