strop-cli 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- strop/__init__.py +5 -0
- strop/__main__.py +5 -0
- strop/analysis/__init__.py +17 -0
- strop/analysis/classify.py +52 -0
- strop/analysis/evidence.py +52 -0
- strop/analysis/facts.py +74 -0
- strop/analysis/patterns.py +90 -0
- strop/cli/__init__.py +5 -0
- strop/cli/analyze.py +93 -0
- strop/cli/app.py +82 -0
- strop/cli/data.py +111 -0
- strop/cli/evaluate.py +145 -0
- strop/cli/export.py +74 -0
- strop/cli/final.py +106 -0
- strop/cli/init.py +131 -0
- strop/cli/optimize.py +156 -0
- strop/cli/options.py +20 -0
- strop/cli/progress.py +18 -0
- strop/cli/prompts.py +110 -0
- strop/cli/status.py +68 -0
- strop/cli/versions.py +123 -0
- strop/config/__init__.py +16 -0
- strop/config/base.py +9 -0
- strop/config/constraints.py +33 -0
- strop/config/dataset.py +63 -0
- strop/config/enums.py +52 -0
- strop/config/io.py +52 -0
- strop/config/limits.py +19 -0
- strop/config/llm.py +42 -0
- strop/config/metrics.py +86 -0
- strop/config/optimization.py +23 -0
- strop/config/output.py +31 -0
- strop/config/overrides.py +22 -0
- strop/config/root.py +36 -0
- strop/config/rules.py +67 -0
- strop/config/split.py +23 -0
- strop/data/__init__.py +58 -0
- strop/data/loaders.py +85 -0
- strop/data/models.py +44 -0
- strop/data/rows.py +99 -0
- strop/data/splitting.py +103 -0
- strop/env.py +44 -0
- strop/errors.py +59 -0
- strop/evaluation/__init__.py +1 -0
- strop/evaluation/budget.py +67 -0
- strop/evaluation/budgeted.py +23 -0
- strop/evaluation/examples.py +49 -0
- strop/evaluation/models.py +105 -0
- strop/evaluation/parsing.py +96 -0
- strop/evaluation/runner.py +107 -0
- strop/formatting.py +14 -0
- strop/llm/__init__.py +7 -0
- strop/llm/caching.py +28 -0
- strop/llm/client.py +15 -0
- strop/llm/factory.py +14 -0
- strop/llm/litellm_client.py +90 -0
- strop/llm/mock.py +43 -0
- strop/llm/retry.py +44 -0
- strop/llm/structured.py +52 -0
- strop/llm/types.py +59 -0
- strop/log.py +49 -0
- strop/metrics/__init__.py +1 -0
- strop/metrics/basic.py +30 -0
- strop/metrics/classification.py +164 -0
- strop/metrics/judge.py +130 -0
- strop/metrics/python_metric.py +101 -0
- strop/metrics/python_runner.py +65 -0
- strop/metrics/registry.py +115 -0
- strop/metrics/rules.py +55 -0
- strop/metrics/structured.py +109 -0
- strop/metrics/suite.py +109 -0
- strop/metrics/text.py +55 -0
- strop/metrics/types.py +106 -0
- strop/optimization/__init__.py +25 -0
- strop/optimization/acceptance.py +170 -0
- strop/optimization/generator.py +92 -0
- strop/optimization/meta_prompt.py +101 -0
- strop/optimization/types.py +69 -0
- strop/prompts/__init__.py +18 -0
- strop/prompts/diff.py +139 -0
- strop/prompts/files.py +92 -0
- strop/prompts/models.py +51 -0
- strop/prompts/render.py +16 -0
- strop/prompts/template.py +45 -0
- strop/prompts/validation.py +31 -0
- strop/prompts/versions.py +34 -0
- strop/py.typed +0 -0
- strop/reporting/__init__.py +6 -0
- strop/reporting/console.py +20 -0
- strop/reporting/diff_view.py +102 -0
- strop/reporting/evaluation_view.py +141 -0
- strop/reporting/final_view.py +109 -0
- strop/reporting/prompt_export.py +58 -0
- strop/reporting/report_markdown.py +157 -0
- strop/reporting/run_view.py +118 -0
- strop/services/__init__.py +1 -0
- strop/services/analysis.py +124 -0
- strop/services/clean.py +38 -0
- strop/services/compare.py +60 -0
- strop/services/context.py +70 -0
- strop/services/data.py +65 -0
- strop/services/evaluate.py +105 -0
- strop/services/evaluator.py +169 -0
- strop/services/export.py +90 -0
- strop/services/final_test.py +69 -0
- strop/services/metrics.py +89 -0
- strop/services/optimize.py +187 -0
- strop/services/partitions.py +46 -0
- strop/services/project.py +55 -0
- strop/services/prompts.py +76 -0
- strop/services/report.py +194 -0
- strop/services/review.py +43 -0
- strop/services/scaffold.py +79 -0
- strop/services/turn.py +234 -0
- strop/storage/__init__.py +6 -0
- strop/storage/cache.py +45 -0
- strop/storage/database.py +45 -0
- strop/storage/datasets.py +129 -0
- strop/storage/evaluations.py +160 -0
- strop/storage/migrations/001_initial.sql +5 -0
- strop/storage/migrations/002_datasets.sql +27 -0
- strop/storage/migrations/003_prompt_versions.sql +22 -0
- strop/storage/migrations/004_llm_cache.sql +6 -0
- strop/storage/migrations/005_evaluations.sql +37 -0
- strop/storage/migrations/006_runs.sql +52 -0
- strop/storage/migrations/007_example_feedback.sql +2 -0
- strop/storage/migrations/__init__.py +5 -0
- strop/storage/migrator.py +91 -0
- strop/storage/paths.py +30 -0
- strop/storage/prompts.py +96 -0
- strop/storage/run_models.py +75 -0
- strop/storage/runs.py +263 -0
- strop_cli-0.1.0.dist-info/METADATA +242 -0
- strop_cli-0.1.0.dist-info/RECORD +137 -0
- strop_cli-0.1.0.dist-info/WHEEL +4 -0
- strop_cli-0.1.0.dist-info/entry_points.txt +2 -0
- strop_cli-0.1.0.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,157 @@
|
|
|
1
|
+
"""Rendering the report dictionary as a Markdown document."""
|
|
2
|
+
|
|
3
|
+
from typing import Any
|
|
4
|
+
|
|
5
|
+
from strop.formatting import fmt
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def render_markdown(report: dict[str, Any]) -> str:
|
|
9
|
+
"""A readable report. Everything here comes from the stored results; nothing is estimated."""
|
|
10
|
+
sections = [
|
|
11
|
+
_title(report),
|
|
12
|
+
_results(report),
|
|
13
|
+
_changes(report),
|
|
14
|
+
_history(report),
|
|
15
|
+
_runs(report),
|
|
16
|
+
_errors(report),
|
|
17
|
+
_usage(report),
|
|
18
|
+
_limitations(report),
|
|
19
|
+
]
|
|
20
|
+
return "\n\n".join(s for s in sections if s) + "\n"
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def _title(report: dict[str, Any]) -> str:
|
|
24
|
+
project, prompt = report["project"], report["prompt"]
|
|
25
|
+
lines = [
|
|
26
|
+
f"# Strop report: {project['name']}",
|
|
27
|
+
"",
|
|
28
|
+
f"Task: {project['task']}",
|
|
29
|
+
"",
|
|
30
|
+
f"Selected prompt: v{prompt['selected']} (initial v{prompt['initial']}, "
|
|
31
|
+
f"best v{prompt['best']}). Generated {report['generated_at']} by Strop "
|
|
32
|
+
f"{report['strop_version']}.",
|
|
33
|
+
]
|
|
34
|
+
models = report["models"]
|
|
35
|
+
lines.append("")
|
|
36
|
+
lines.append(
|
|
37
|
+
"Models: " + ", ".join(f"{role} `{name}`" for role, name in models.items() if name)
|
|
38
|
+
)
|
|
39
|
+
return "\n".join(lines)
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def _results(report: dict[str, Any]) -> str:
|
|
43
|
+
primary = report["metrics"]["primary"]
|
|
44
|
+
rows = []
|
|
45
|
+
for label, key in (
|
|
46
|
+
("Initial prompt", "initial_validation"),
|
|
47
|
+
("Selected prompt", "selected_validation"),
|
|
48
|
+
("Initial prompt", "initial_test"),
|
|
49
|
+
("Selected prompt", "selected_test"),
|
|
50
|
+
):
|
|
51
|
+
record = report["results"][key]
|
|
52
|
+
if record is None:
|
|
53
|
+
continue
|
|
54
|
+
value = record["metrics"][primary]["value"]
|
|
55
|
+
rows.append(
|
|
56
|
+
f"| {label} (v{record['version'].lstrip('v')}) | {record['partition']} | "
|
|
57
|
+
f"{fmt(value)} | {record['samples']} |"
|
|
58
|
+
)
|
|
59
|
+
if not rows:
|
|
60
|
+
return "## Results\n\nNo evaluations are stored yet."
|
|
61
|
+
header = f"| Prompt | Partition | {primary} | Examples |\n| --- | --- | --- | --- |"
|
|
62
|
+
return "## Results\n\n" + header + "\n" + "\n".join(rows)
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def _changes(report: dict[str, Any]) -> str:
|
|
66
|
+
diff = report["diff"]
|
|
67
|
+
if not diff["changed"]:
|
|
68
|
+
return "## Prompt changes\n\nThe selected prompt is the initial prompt."
|
|
69
|
+
return "## Prompt changes\n\n```diff\n" + _diff_lines(diff) + "\n```"
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def _diff_lines(diff: dict[str, Any]) -> str:
|
|
73
|
+
out = []
|
|
74
|
+
for message in diff["messages"]:
|
|
75
|
+
out.append(f"# message {message['index']} ({message['role']}): {message['status']}")
|
|
76
|
+
for line in message["lines"]:
|
|
77
|
+
prefix = {"add": "+", "remove": "-", "context": " ", "hunk": ""}[line["kind"]]
|
|
78
|
+
out.append(f"{prefix}{line['text']}")
|
|
79
|
+
return "\n".join(out)
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def _history(report: dict[str, Any]) -> str:
|
|
83
|
+
lines = [
|
|
84
|
+
"## Version history",
|
|
85
|
+
"",
|
|
86
|
+
"| Version | Parent | Source | Decision | Change | Summary |",
|
|
87
|
+
"| --- | --- | --- | --- | --- | --- |",
|
|
88
|
+
]
|
|
89
|
+
for v in report["versions"]:
|
|
90
|
+
delta = "" if v["primary_delta"] is None else f"{v['primary_delta']:+.4f}"
|
|
91
|
+
parent = f"v{v['parent']}" if v["parent"] else ""
|
|
92
|
+
lines.append(
|
|
93
|
+
f"| v{v['version']} | {parent} | {v['source']} | {v['status'] or ''} | {delta} | "
|
|
94
|
+
f"{_cell(v['change_summary'])} |"
|
|
95
|
+
)
|
|
96
|
+
return "\n".join(lines)
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def _runs(report: dict[str, Any]) -> str:
|
|
100
|
+
if not report["runs"]:
|
|
101
|
+
return ""
|
|
102
|
+
lines = [
|
|
103
|
+
"## Optimization runs",
|
|
104
|
+
"",
|
|
105
|
+
"| Run | Status | Stopped because | Turns | Model calls |",
|
|
106
|
+
"| --- | --- | --- | --- | --- |",
|
|
107
|
+
]
|
|
108
|
+
lines += [
|
|
109
|
+
f"| {r['id']} | {r['status']} | {r['stop_reason'] or ''} | {r['turns']} | "
|
|
110
|
+
f"{r['model_calls']} |"
|
|
111
|
+
for r in report["runs"]
|
|
112
|
+
]
|
|
113
|
+
return "\n".join(lines)
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
def _errors(report: dict[str, Any]) -> str:
|
|
117
|
+
analysis = report["error_analysis"]
|
|
118
|
+
if analysis is None:
|
|
119
|
+
return ""
|
|
120
|
+
lines = [
|
|
121
|
+
"## Errors on validation",
|
|
122
|
+
"",
|
|
123
|
+
f"{analysis['failed']} of {analysis['examples']} examples failed (counts only).",
|
|
124
|
+
]
|
|
125
|
+
lines += [f"- {code}: {count}" for code, count in sorted(analysis["codes"].items())]
|
|
126
|
+
per_class = report["per_class"]
|
|
127
|
+
if per_class:
|
|
128
|
+
lines += [
|
|
129
|
+
"",
|
|
130
|
+
"| Class | Precision | Recall | F1 | Support |",
|
|
131
|
+
"| --- | --- | --- | --- | --- |",
|
|
132
|
+
]
|
|
133
|
+
lines += [
|
|
134
|
+
f"| {name} | {fmt(s['precision'])} | {fmt(s['recall'])} | {fmt(s['f1'])} | "
|
|
135
|
+
f"{s['support']} |"
|
|
136
|
+
for name, s in per_class.items()
|
|
137
|
+
]
|
|
138
|
+
return "\n".join(lines)
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
def _usage(report: dict[str, Any]) -> str:
|
|
142
|
+
usage = report["usage"]
|
|
143
|
+
cost = "unavailable" if usage["cost_usd"] is None else f"${usage['cost_usd']:.4f}"
|
|
144
|
+
tokens = "unavailable" if usage["total_tokens"] is None else f"{usage['total_tokens']:,}"
|
|
145
|
+
return (
|
|
146
|
+
"## Usage\n\n"
|
|
147
|
+
f"{usage['evaluations']} evaluations, {usage['model_calls']} model calls, "
|
|
148
|
+
f"tokens {tokens}, cost {cost}."
|
|
149
|
+
)
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
def _limitations(report: dict[str, Any]) -> str:
|
|
153
|
+
return "## Limitations\n\n" + "\n".join(f"- {note}" for note in report["limitations"])
|
|
154
|
+
|
|
155
|
+
|
|
156
|
+
def _cell(text: str) -> str:
|
|
157
|
+
return text.replace("|", "/").replace("\n", " ")
|
|
@@ -0,0 +1,118 @@
|
|
|
1
|
+
"""Terminal rendering and JSON payloads for optimization runs."""
|
|
2
|
+
|
|
3
|
+
from typing import Any
|
|
4
|
+
|
|
5
|
+
from rich.console import Console
|
|
6
|
+
from rich.table import Table
|
|
7
|
+
from rich.text import Text
|
|
8
|
+
|
|
9
|
+
from strop.formatting import fmt, shorten
|
|
10
|
+
from strop.prompts.diff import diff_templates
|
|
11
|
+
from strop.reporting.diff_view import render_diff
|
|
12
|
+
from strop.services.optimize import RunSummary
|
|
13
|
+
from strop.storage.runs import CandidateRecord
|
|
14
|
+
|
|
15
|
+
_STOP_TEXT = {
|
|
16
|
+
"max_turns": "reached the maximum number of turns",
|
|
17
|
+
"target_reached": "reached the target metric",
|
|
18
|
+
"patience": "no accepted improvement for the configured number of turns",
|
|
19
|
+
"no_failures": "no failures left on the training examples",
|
|
20
|
+
"budget": "stopped by a budget",
|
|
21
|
+
"awaiting_approval": "paused for approval",
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def stop_text(summary: RunSummary) -> str:
|
|
26
|
+
text = _STOP_TEXT.get(summary.stop_reason, summary.stop_reason)
|
|
27
|
+
if summary.stop_detail:
|
|
28
|
+
text += f" ({summary.stop_detail})"
|
|
29
|
+
if summary.pending is not None:
|
|
30
|
+
text += (
|
|
31
|
+
f": v{summary.pending.version_number} is waiting for 'strop approve' or 'strop reject'"
|
|
32
|
+
)
|
|
33
|
+
return text
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def render_run(console: Console, summary: RunSummary) -> None:
|
|
37
|
+
"""Print the run result: why it stopped, the metric change, candidates, and the prompt diff."""
|
|
38
|
+
console.print(
|
|
39
|
+
Text(f"Run {summary.run.id}: {stop_text(summary)}.", style="bold"), soft_wrap=True
|
|
40
|
+
)
|
|
41
|
+
primary = _primary(summary)
|
|
42
|
+
before, after = _value(summary.baseline), _value(summary.result)
|
|
43
|
+
change = (
|
|
44
|
+
f"{summary.start.ref} {primary} {fmt(before)} -> "
|
|
45
|
+
f"{summary.final.ref} {primary} {fmt(after)} (validation)"
|
|
46
|
+
)
|
|
47
|
+
console.print(change, soft_wrap=True, highlight=False)
|
|
48
|
+
if summary.candidates:
|
|
49
|
+
console.print(_candidate_table(summary.candidates))
|
|
50
|
+
cost = "unavailable" if summary.cost_usd is None else f"${summary.cost_usd:.4f}"
|
|
51
|
+
console.print(Text(f"Model calls: {summary.calls} | Cost: {cost}", style="dim"), soft_wrap=True)
|
|
52
|
+
for warning in summary.warnings:
|
|
53
|
+
console.print(f"Warning: {warning}", soft_wrap=True, markup=False, highlight=False)
|
|
54
|
+
_render_final_diff(console, summary)
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def _render_final_diff(console: Console, summary: RunSummary) -> None:
|
|
58
|
+
if summary.final.number == summary.initial.number:
|
|
59
|
+
console.print("No improvement was accepted; the best prompt is unchanged.", soft_wrap=True)
|
|
60
|
+
return
|
|
61
|
+
diff = diff_templates(summary.initial.template, summary.final.template)
|
|
62
|
+
console.print(
|
|
63
|
+
Text(f"Prompt changes, {summary.initial.ref} -> {summary.final.ref}:", style="bold"),
|
|
64
|
+
soft_wrap=True,
|
|
65
|
+
)
|
|
66
|
+
render_diff(diff, console)
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def _candidate_table(candidates: list[CandidateRecord]) -> Table:
|
|
70
|
+
table = Table(box=None, pad_edge=False, header_style="bold")
|
|
71
|
+
for column in ("Turn", "Version", "Result", "Change", "Summary"):
|
|
72
|
+
table.add_column(column)
|
|
73
|
+
for item in candidates:
|
|
74
|
+
delta = "-" if item.primary_delta is None else f"{item.primary_delta:+.4f}"
|
|
75
|
+
version = f"v{item.version_number}" if item.version_number else "-"
|
|
76
|
+
note = item.change_summary or item.reason
|
|
77
|
+
table.add_row(str(item.turn), version, item.status, delta, shorten(note, 60))
|
|
78
|
+
return table
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def _primary(summary: RunSummary) -> str:
|
|
82
|
+
record = summary.result or summary.baseline
|
|
83
|
+
return record.report.primary if record else "primary metric"
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def _value(record: Any) -> float | None:
|
|
87
|
+
return record.report.primary_value.value if record is not None else None
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def run_payload(summary: RunSummary) -> dict[str, Any]:
|
|
91
|
+
return {
|
|
92
|
+
"run": summary.run.id,
|
|
93
|
+
"status": summary.run.status,
|
|
94
|
+
"stop_reason": summary.stop_reason,
|
|
95
|
+
"stop_detail": summary.stop_detail,
|
|
96
|
+
"initial_version": summary.initial.number,
|
|
97
|
+
"start_version": summary.start.number,
|
|
98
|
+
"best_version": summary.final.number,
|
|
99
|
+
"primary_metric": _primary(summary),
|
|
100
|
+
"start_value": _value(summary.baseline),
|
|
101
|
+
"best_value": _value(summary.result),
|
|
102
|
+
"turns": len([t for t in summary.turns if t.status == "done"]),
|
|
103
|
+
"pending_version": summary.pending.version_number if summary.pending else None,
|
|
104
|
+
"model_calls": summary.calls,
|
|
105
|
+
"cost_usd": summary.cost_usd,
|
|
106
|
+
"candidates": [
|
|
107
|
+
{
|
|
108
|
+
"turn": c.turn,
|
|
109
|
+
"version": c.version_number,
|
|
110
|
+
"status": c.status,
|
|
111
|
+
"primary_delta": c.primary_delta,
|
|
112
|
+
"summary": c.change_summary,
|
|
113
|
+
"reason": c.reason,
|
|
114
|
+
}
|
|
115
|
+
for c in summary.candidates
|
|
116
|
+
],
|
|
117
|
+
"diff": diff_templates(summary.initial.template, summary.final.template).to_payload(),
|
|
118
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""Application workflows used by the command-line layer."""
|
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
"""Analyzing the failures of one prompt version on the train partition."""
|
|
2
|
+
|
|
3
|
+
import sqlite3
|
|
4
|
+
from dataclasses import dataclass
|
|
5
|
+
from typing import Any
|
|
6
|
+
|
|
7
|
+
from strop.analysis import (
|
|
8
|
+
AnalysisFacts,
|
|
9
|
+
FailedExample,
|
|
10
|
+
Pattern,
|
|
11
|
+
build_facts,
|
|
12
|
+
find_patterns,
|
|
13
|
+
select_evidence,
|
|
14
|
+
)
|
|
15
|
+
from strop.config.enums import ModelRole
|
|
16
|
+
from strop.config.llm import ModelSpec
|
|
17
|
+
from strop.config.optimization import OptimizationConfig
|
|
18
|
+
from strop.config.overrides import apply_overrides
|
|
19
|
+
from strop.data.models import Partition
|
|
20
|
+
from strop.errors import InvalidModelReplyError
|
|
21
|
+
from strop.evaluation.budget import BudgetTracker
|
|
22
|
+
from strop.evaluation.budgeted import BudgetedClient
|
|
23
|
+
from strop.evaluation.models import EvaluationRecord
|
|
24
|
+
from strop.llm.client import LLMClient
|
|
25
|
+
from strop.llm.factory import build_client
|
|
26
|
+
from strop.prompts.versions import PromptVersion
|
|
27
|
+
from strop.services.evaluate import sample_rows
|
|
28
|
+
from strop.services.evaluator import Evaluator, RunTag, build_evaluator
|
|
29
|
+
from strop.services.project import Project
|
|
30
|
+
from strop.services.prompts import ensure_initial_version, resolve_version
|
|
31
|
+
from strop.storage.cache import ResponseCache
|
|
32
|
+
from strop.storage.prompts import PromptRepository
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
@dataclass(frozen=True)
|
|
36
|
+
class TurnAnalysis:
|
|
37
|
+
train_record: EvaluationRecord
|
|
38
|
+
facts: AnalysisFacts
|
|
39
|
+
evidence: tuple[FailedExample, ...]
|
|
40
|
+
patterns: tuple[Pattern, ...]
|
|
41
|
+
|
|
42
|
+
def to_json(self) -> dict[str, Any]:
|
|
43
|
+
"""The analysis as stored with a turn. Patterns are labeled as hypotheses."""
|
|
44
|
+
return {
|
|
45
|
+
"facts": {
|
|
46
|
+
"examples": self.facts.total,
|
|
47
|
+
"failed": self.facts.failed,
|
|
48
|
+
"codes": dict(self.facts.codes),
|
|
49
|
+
"confusions": [
|
|
50
|
+
{"expected": e, "got": g, "count": n} for e, g, n in self.facts.confusions
|
|
51
|
+
],
|
|
52
|
+
},
|
|
53
|
+
"hypotheses": [
|
|
54
|
+
{"name": p.name, "description": p.description, "examples": list(p.example_ids)}
|
|
55
|
+
for p in self.patterns
|
|
56
|
+
],
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
async def analyze_version(
|
|
61
|
+
evaluator: Evaluator,
|
|
62
|
+
version: PromptVersion,
|
|
63
|
+
settings: OptimizationConfig,
|
|
64
|
+
*,
|
|
65
|
+
seed: int,
|
|
66
|
+
analyzer: tuple[LLMClient, ModelSpec] | None,
|
|
67
|
+
run_id: int | None = None,
|
|
68
|
+
) -> TurnAnalysis:
|
|
69
|
+
"""Evaluate ``version`` on train, then collect facts, evidence, and (optionally) patterns."""
|
|
70
|
+
rows = evaluator.rows(Partition.TRAIN)
|
|
71
|
+
sampled = settings.train_sample is not None and settings.train_sample < len(rows)
|
|
72
|
+
if sampled and settings.train_sample is not None:
|
|
73
|
+
rows = sample_rows(rows, settings.train_sample, seed)
|
|
74
|
+
record = None if sampled else evaluator.reusable(version, Partition.TRAIN)
|
|
75
|
+
if record is None:
|
|
76
|
+
tag = RunTag("analysis", {"sampled": sampled}, run_id)
|
|
77
|
+
record = await evaluator.evaluate(version, Partition.TRAIN, tag, rows=rows)
|
|
78
|
+
outcomes = evaluator.repository.outcomes(record.id)
|
|
79
|
+
config = evaluator.project.config
|
|
80
|
+
facts = build_facts(outcomes, rows, config.output.type, evaluator.suite.primary)
|
|
81
|
+
evidence = select_evidence(facts.failures, settings.evidence_examples, seed)
|
|
82
|
+
patterns: list[Pattern] = []
|
|
83
|
+
if analyzer is not None and evidence:
|
|
84
|
+
client, spec = analyzer
|
|
85
|
+
try:
|
|
86
|
+
patterns = await find_patterns(
|
|
87
|
+
client,
|
|
88
|
+
spec,
|
|
89
|
+
task=config.project.task_description,
|
|
90
|
+
output=config.output,
|
|
91
|
+
facts=facts,
|
|
92
|
+
evidence=evidence,
|
|
93
|
+
max_chars=settings.evidence_chars,
|
|
94
|
+
)
|
|
95
|
+
except InvalidModelReplyError:
|
|
96
|
+
patterns = []
|
|
97
|
+
return TurnAnalysis(record, facts, tuple(evidence), tuple(patterns))
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
async def run_analysis(
|
|
101
|
+
project: Project,
|
|
102
|
+
conn: sqlite3.Connection,
|
|
103
|
+
ref: str | None,
|
|
104
|
+
*,
|
|
105
|
+
patterns: bool,
|
|
106
|
+
examples: int | None,
|
|
107
|
+
use_cache: bool,
|
|
108
|
+
) -> tuple[PromptVersion, TurnAnalysis]:
|
|
109
|
+
"""Analyze the train failures of a prompt version, optionally asking for patterns."""
|
|
110
|
+
repo = PromptRepository(conn)
|
|
111
|
+
ensure_initial_version(project, repo)
|
|
112
|
+
version = resolve_version(repo, ref)
|
|
113
|
+
settings = apply_overrides(project.config.optimization, {"evidence_examples": examples})
|
|
114
|
+
budget = BudgetTracker(project.config.budgets)
|
|
115
|
+
evaluator = build_evaluator(project, conn, use_cache=use_cache, budget=budget)
|
|
116
|
+
analyzer = None
|
|
117
|
+
if patterns:
|
|
118
|
+
cache = ResponseCache(conn) if use_cache else None
|
|
119
|
+
client = BudgetedClient(build_client(project.config.execution, cache), budget)
|
|
120
|
+
analyzer = (client, project.config.models.resolve(ModelRole.ANALYZER))
|
|
121
|
+
analysis = await analyze_version(
|
|
122
|
+
evaluator, version, settings, seed=evaluator.partitions.seed, analyzer=analyzer
|
|
123
|
+
)
|
|
124
|
+
return version, analysis
|
strop/services/clean.py
ADDED
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
"""Deleting stored data on request."""
|
|
2
|
+
|
|
3
|
+
import shutil
|
|
4
|
+
|
|
5
|
+
from strop.errors import ConfigError
|
|
6
|
+
from strop.services.project import Project
|
|
7
|
+
from strop.storage.cache import ResponseCache
|
|
8
|
+
|
|
9
|
+
CHOICES = "--cache, --outputs, or --all"
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def clean_project(
|
|
13
|
+
project: Project, *, cache: bool, outputs: bool, everything: bool, confirmed: bool
|
|
14
|
+
) -> list[str]:
|
|
15
|
+
"""Delete the chosen data and report what was removed; nothing goes without ``confirmed``."""
|
|
16
|
+
if not (cache or outputs or everything):
|
|
17
|
+
raise ConfigError(f"choose what to delete: {CHOICES}")
|
|
18
|
+
if not confirmed:
|
|
19
|
+
raise ConfigError("this permanently deletes stored data; pass --yes to confirm")
|
|
20
|
+
messages = []
|
|
21
|
+
if everything:
|
|
22
|
+
shutil.rmtree(project.paths.state_dir, ignore_errors=True)
|
|
23
|
+
return [
|
|
24
|
+
f"Deleted {project.paths.state_dir.name}/ (database, cache, prompt versions, results)",
|
|
25
|
+
"strop.yaml, the prompt file, and the dataset were not touched.",
|
|
26
|
+
]
|
|
27
|
+
with project.database() as conn:
|
|
28
|
+
if cache or outputs:
|
|
29
|
+
removed = ResponseCache(conn).clear()
|
|
30
|
+
messages.append(f"Deleted {removed} cached model response(s).")
|
|
31
|
+
if outputs:
|
|
32
|
+
cleared = conn.execute(
|
|
33
|
+
"UPDATE example_results SET raw_output = '', value = NULL, parsed_json = NULL, "
|
|
34
|
+
"detail = NULL, feedback = NULL"
|
|
35
|
+
).rowcount
|
|
36
|
+
conn.execute("UPDATE candidates SET raw_response = ''")
|
|
37
|
+
messages.append(f"Cleared the stored model output of {cleared} example result(s).")
|
|
38
|
+
return messages
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
"""Running one prompt version on several target models."""
|
|
2
|
+
|
|
3
|
+
import sqlite3
|
|
4
|
+
from dataclasses import dataclass, replace
|
|
5
|
+
|
|
6
|
+
from strop.data.models import Partition
|
|
7
|
+
from strop.errors import ConfigError
|
|
8
|
+
from strop.evaluation.models import EvaluationRecord
|
|
9
|
+
from strop.evaluation.runner import Progress
|
|
10
|
+
from strop.prompts.versions import PromptVersion
|
|
11
|
+
from strop.services.evaluate import sample_rows
|
|
12
|
+
from strop.services.evaluator import RunTag, build_evaluator
|
|
13
|
+
from strop.services.project import Project
|
|
14
|
+
from strop.services.prompts import ensure_initial_version, resolve_version
|
|
15
|
+
from strop.storage.prompts import PromptRepository
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
@dataclass(frozen=True)
|
|
19
|
+
class Comparison:
|
|
20
|
+
version: PromptVersion
|
|
21
|
+
partition: Partition
|
|
22
|
+
results: list[tuple[str, EvaluationRecord]]
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def parse_models(text: str) -> list[str]:
|
|
26
|
+
"""Split a comma-separated list of model names, rejecting empty and repeated entries."""
|
|
27
|
+
names = [item.strip() for item in text.split(",") if item.strip()]
|
|
28
|
+
if len(names) < 2:
|
|
29
|
+
raise ConfigError("--models needs at least two comma-separated model names")
|
|
30
|
+
if len(set(names)) != len(names):
|
|
31
|
+
raise ConfigError("--models lists the same model more than once")
|
|
32
|
+
return names
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
async def run_compare( # noqa: PLR0913
|
|
36
|
+
project: Project,
|
|
37
|
+
conn: sqlite3.Connection,
|
|
38
|
+
*,
|
|
39
|
+
models: list[str],
|
|
40
|
+
ref: str | None,
|
|
41
|
+
partition: Partition,
|
|
42
|
+
sample: int | None,
|
|
43
|
+
use_cache: bool,
|
|
44
|
+
progress: Progress | None = None,
|
|
45
|
+
) -> Comparison:
|
|
46
|
+
"""Evaluate the same version on the same rows with each model as the target."""
|
|
47
|
+
repo = PromptRepository(conn)
|
|
48
|
+
ensure_initial_version(project, repo)
|
|
49
|
+
version = resolve_version(repo, ref)
|
|
50
|
+
base = build_evaluator(project, conn, use_cache=use_cache)
|
|
51
|
+
rows = base.rows(partition)
|
|
52
|
+
if sample is not None:
|
|
53
|
+
rows = sample_rows(rows, sample, base.partitions.seed)
|
|
54
|
+
results = []
|
|
55
|
+
for name in models:
|
|
56
|
+
evaluator = replace(base, target=base.target.model_copy(update={"model": name}))
|
|
57
|
+
tag = RunTag("compare", {"sampled": sample is not None})
|
|
58
|
+
record = await evaluator.evaluate(version, partition, tag, rows=rows, progress=progress)
|
|
59
|
+
results.append((name, record))
|
|
60
|
+
return Comparison(version, partition, results)
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
"""Everything the optimization loop needs, assembled once per command."""
|
|
2
|
+
|
|
3
|
+
import sqlite3
|
|
4
|
+
from collections.abc import Callable
|
|
5
|
+
from dataclasses import dataclass
|
|
6
|
+
|
|
7
|
+
from strop.config.enums import ModelRole
|
|
8
|
+
from strop.config.limits import BudgetsConfig
|
|
9
|
+
from strop.config.llm import ModelSpec
|
|
10
|
+
from strop.config.optimization import OptimizationConfig
|
|
11
|
+
from strop.evaluation.budget import BudgetTracker
|
|
12
|
+
from strop.evaluation.budgeted import BudgetedClient
|
|
13
|
+
from strop.llm.client import LLMClient
|
|
14
|
+
from strop.llm.factory import build_client
|
|
15
|
+
from strop.optimization import AcceptanceRules, CandidateGenerator
|
|
16
|
+
from strop.services.evaluator import Evaluator, build_evaluator
|
|
17
|
+
from strop.services.project import Project
|
|
18
|
+
from strop.storage.cache import ResponseCache
|
|
19
|
+
from strop.storage.prompts import PromptRepository
|
|
20
|
+
from strop.storage.runs import RunRepository
|
|
21
|
+
|
|
22
|
+
Say = Callable[[str], None]
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
@dataclass(frozen=True)
|
|
26
|
+
class LoopContext:
|
|
27
|
+
project: Project
|
|
28
|
+
settings: OptimizationConfig
|
|
29
|
+
evaluator: Evaluator
|
|
30
|
+
prompts: PromptRepository
|
|
31
|
+
runs: RunRepository
|
|
32
|
+
generator: CandidateGenerator
|
|
33
|
+
analyzer_client: LLMClient
|
|
34
|
+
analyzer_spec: ModelSpec
|
|
35
|
+
rules: AcceptanceRules
|
|
36
|
+
say: Say
|
|
37
|
+
|
|
38
|
+
@property
|
|
39
|
+
def budget(self) -> BudgetTracker:
|
|
40
|
+
return self.evaluator.budget
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def build_context(
|
|
44
|
+
project: Project,
|
|
45
|
+
conn: sqlite3.Connection,
|
|
46
|
+
settings: OptimizationConfig,
|
|
47
|
+
budgets: BudgetsConfig,
|
|
48
|
+
*,
|
|
49
|
+
use_cache: bool,
|
|
50
|
+
say: Say,
|
|
51
|
+
) -> LoopContext:
|
|
52
|
+
"""Wire the evaluator, optimizer, analyzer and acceptance rules to one shared budget."""
|
|
53
|
+
config = project.config
|
|
54
|
+
budget = BudgetTracker(budgets)
|
|
55
|
+
evaluator = build_evaluator(project, conn, use_cache=use_cache, budget=budget)
|
|
56
|
+
cache = ResponseCache(conn) if use_cache else None
|
|
57
|
+
helper = BudgetedClient(build_client(config.execution, cache), budget)
|
|
58
|
+
optimizer = config.models.resolve(ModelRole.OPTIMIZER)
|
|
59
|
+
return LoopContext(
|
|
60
|
+
project=project,
|
|
61
|
+
settings=settings,
|
|
62
|
+
evaluator=evaluator,
|
|
63
|
+
prompts=PromptRepository(conn),
|
|
64
|
+
runs=RunRepository(conn),
|
|
65
|
+
generator=CandidateGenerator(helper, optimizer),
|
|
66
|
+
analyzer_client=helper,
|
|
67
|
+
analyzer_spec=config.models.resolve(ModelRole.ANALYZER),
|
|
68
|
+
rules=AcceptanceRules.from_config(settings, config.metrics.constraints),
|
|
69
|
+
say=say,
|
|
70
|
+
)
|
strop/services/data.py
ADDED
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
"""Dataset workflows: validation and creating persisted splits."""
|
|
2
|
+
|
|
3
|
+
from dataclasses import dataclass
|
|
4
|
+
from typing import Any
|
|
5
|
+
|
|
6
|
+
from strop.config.overrides import apply_overrides
|
|
7
|
+
from strop.config.split import SplitConfig
|
|
8
|
+
from strop.data import Dataset, SplitSummary, load_dataset, require_valid, split_rows, summarize
|
|
9
|
+
from strop.data.splitting import stratify_groups
|
|
10
|
+
from strop.services.project import Project
|
|
11
|
+
from strop.storage.datasets import DatasetRepository
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
@dataclass(frozen=True)
|
|
15
|
+
class SplitOutcome:
|
|
16
|
+
split_id: int
|
|
17
|
+
created: bool
|
|
18
|
+
dataset: Dataset
|
|
19
|
+
config: SplitConfig
|
|
20
|
+
summary: SplitSummary
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def check_dataset(project: Project) -> Dataset:
|
|
24
|
+
"""Load and validate the dataset without raising on invalid rows."""
|
|
25
|
+
return load_dataset(project.config, project.root)
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def effective_split_config(project: Project, overrides: dict[str, Any]) -> SplitConfig:
|
|
29
|
+
"""The configured split settings with any non-None command-line overrides applied."""
|
|
30
|
+
return apply_overrides(project.config.split, overrides)
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def create_split(project: Project, split: SplitConfig, *, skip_invalid: bool) -> SplitOutcome:
|
|
34
|
+
"""Split the dataset and persist it. An identical existing split is reused, not recomputed."""
|
|
35
|
+
dataset = require_valid(check_dataset(project), skip_invalid=skip_invalid)
|
|
36
|
+
conn = project.connect()
|
|
37
|
+
try:
|
|
38
|
+
repo = DatasetRepository(conn)
|
|
39
|
+
dataset_id = repo.save_dataset(
|
|
40
|
+
content_hash=dataset.content_hash,
|
|
41
|
+
source_path=str(project.config.dataset.path),
|
|
42
|
+
row_count=len(dataset.rows),
|
|
43
|
+
skipped_rows=dataset.total_records - len(dataset.rows),
|
|
44
|
+
skip_invalid=skip_invalid,
|
|
45
|
+
)
|
|
46
|
+
config_json = split.model_dump_json()
|
|
47
|
+
split_id = repo.find_split(dataset_id, config_json)
|
|
48
|
+
created = split_id is None
|
|
49
|
+
if split_id is None:
|
|
50
|
+
assignments = split_rows(dataset.rows, split)
|
|
51
|
+
stratified = len(stratify_groups(dataset.rows, split)) > 1
|
|
52
|
+
strategy = "stratified" if stratified else "random"
|
|
53
|
+
split_id = repo.create_split(
|
|
54
|
+
dataset_id=dataset_id,
|
|
55
|
+
seed=split.seed,
|
|
56
|
+
strategy=strategy,
|
|
57
|
+
config_json=config_json,
|
|
58
|
+
assignments=assignments,
|
|
59
|
+
)
|
|
60
|
+
assignments = repo.assignments(split_id)
|
|
61
|
+
finally:
|
|
62
|
+
conn.close()
|
|
63
|
+
return SplitOutcome(
|
|
64
|
+
split_id, created, dataset, split, summarize(dataset.rows, assignments, split)
|
|
65
|
+
)
|