strop-cli 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (137) hide show
  1. strop/__init__.py +5 -0
  2. strop/__main__.py +5 -0
  3. strop/analysis/__init__.py +17 -0
  4. strop/analysis/classify.py +52 -0
  5. strop/analysis/evidence.py +52 -0
  6. strop/analysis/facts.py +74 -0
  7. strop/analysis/patterns.py +90 -0
  8. strop/cli/__init__.py +5 -0
  9. strop/cli/analyze.py +93 -0
  10. strop/cli/app.py +82 -0
  11. strop/cli/data.py +111 -0
  12. strop/cli/evaluate.py +145 -0
  13. strop/cli/export.py +74 -0
  14. strop/cli/final.py +106 -0
  15. strop/cli/init.py +131 -0
  16. strop/cli/optimize.py +156 -0
  17. strop/cli/options.py +20 -0
  18. strop/cli/progress.py +18 -0
  19. strop/cli/prompts.py +110 -0
  20. strop/cli/status.py +68 -0
  21. strop/cli/versions.py +123 -0
  22. strop/config/__init__.py +16 -0
  23. strop/config/base.py +9 -0
  24. strop/config/constraints.py +33 -0
  25. strop/config/dataset.py +63 -0
  26. strop/config/enums.py +52 -0
  27. strop/config/io.py +52 -0
  28. strop/config/limits.py +19 -0
  29. strop/config/llm.py +42 -0
  30. strop/config/metrics.py +86 -0
  31. strop/config/optimization.py +23 -0
  32. strop/config/output.py +31 -0
  33. strop/config/overrides.py +22 -0
  34. strop/config/root.py +36 -0
  35. strop/config/rules.py +67 -0
  36. strop/config/split.py +23 -0
  37. strop/data/__init__.py +58 -0
  38. strop/data/loaders.py +85 -0
  39. strop/data/models.py +44 -0
  40. strop/data/rows.py +99 -0
  41. strop/data/splitting.py +103 -0
  42. strop/env.py +44 -0
  43. strop/errors.py +59 -0
  44. strop/evaluation/__init__.py +1 -0
  45. strop/evaluation/budget.py +67 -0
  46. strop/evaluation/budgeted.py +23 -0
  47. strop/evaluation/examples.py +49 -0
  48. strop/evaluation/models.py +105 -0
  49. strop/evaluation/parsing.py +96 -0
  50. strop/evaluation/runner.py +107 -0
  51. strop/formatting.py +14 -0
  52. strop/llm/__init__.py +7 -0
  53. strop/llm/caching.py +28 -0
  54. strop/llm/client.py +15 -0
  55. strop/llm/factory.py +14 -0
  56. strop/llm/litellm_client.py +90 -0
  57. strop/llm/mock.py +43 -0
  58. strop/llm/retry.py +44 -0
  59. strop/llm/structured.py +52 -0
  60. strop/llm/types.py +59 -0
  61. strop/log.py +49 -0
  62. strop/metrics/__init__.py +1 -0
  63. strop/metrics/basic.py +30 -0
  64. strop/metrics/classification.py +164 -0
  65. strop/metrics/judge.py +130 -0
  66. strop/metrics/python_metric.py +101 -0
  67. strop/metrics/python_runner.py +65 -0
  68. strop/metrics/registry.py +115 -0
  69. strop/metrics/rules.py +55 -0
  70. strop/metrics/structured.py +109 -0
  71. strop/metrics/suite.py +109 -0
  72. strop/metrics/text.py +55 -0
  73. strop/metrics/types.py +106 -0
  74. strop/optimization/__init__.py +25 -0
  75. strop/optimization/acceptance.py +170 -0
  76. strop/optimization/generator.py +92 -0
  77. strop/optimization/meta_prompt.py +101 -0
  78. strop/optimization/types.py +69 -0
  79. strop/prompts/__init__.py +18 -0
  80. strop/prompts/diff.py +139 -0
  81. strop/prompts/files.py +92 -0
  82. strop/prompts/models.py +51 -0
  83. strop/prompts/render.py +16 -0
  84. strop/prompts/template.py +45 -0
  85. strop/prompts/validation.py +31 -0
  86. strop/prompts/versions.py +34 -0
  87. strop/py.typed +0 -0
  88. strop/reporting/__init__.py +6 -0
  89. strop/reporting/console.py +20 -0
  90. strop/reporting/diff_view.py +102 -0
  91. strop/reporting/evaluation_view.py +141 -0
  92. strop/reporting/final_view.py +109 -0
  93. strop/reporting/prompt_export.py +58 -0
  94. strop/reporting/report_markdown.py +157 -0
  95. strop/reporting/run_view.py +118 -0
  96. strop/services/__init__.py +1 -0
  97. strop/services/analysis.py +124 -0
  98. strop/services/clean.py +38 -0
  99. strop/services/compare.py +60 -0
  100. strop/services/context.py +70 -0
  101. strop/services/data.py +65 -0
  102. strop/services/evaluate.py +105 -0
  103. strop/services/evaluator.py +169 -0
  104. strop/services/export.py +90 -0
  105. strop/services/final_test.py +69 -0
  106. strop/services/metrics.py +89 -0
  107. strop/services/optimize.py +187 -0
  108. strop/services/partitions.py +46 -0
  109. strop/services/project.py +55 -0
  110. strop/services/prompts.py +76 -0
  111. strop/services/report.py +194 -0
  112. strop/services/review.py +43 -0
  113. strop/services/scaffold.py +79 -0
  114. strop/services/turn.py +234 -0
  115. strop/storage/__init__.py +6 -0
  116. strop/storage/cache.py +45 -0
  117. strop/storage/database.py +45 -0
  118. strop/storage/datasets.py +129 -0
  119. strop/storage/evaluations.py +160 -0
  120. strop/storage/migrations/001_initial.sql +5 -0
  121. strop/storage/migrations/002_datasets.sql +27 -0
  122. strop/storage/migrations/003_prompt_versions.sql +22 -0
  123. strop/storage/migrations/004_llm_cache.sql +6 -0
  124. strop/storage/migrations/005_evaluations.sql +37 -0
  125. strop/storage/migrations/006_runs.sql +52 -0
  126. strop/storage/migrations/007_example_feedback.sql +2 -0
  127. strop/storage/migrations/__init__.py +5 -0
  128. strop/storage/migrator.py +91 -0
  129. strop/storage/paths.py +30 -0
  130. strop/storage/prompts.py +96 -0
  131. strop/storage/run_models.py +75 -0
  132. strop/storage/runs.py +263 -0
  133. strop_cli-0.1.0.dist-info/METADATA +242 -0
  134. strop_cli-0.1.0.dist-info/RECORD +137 -0
  135. strop_cli-0.1.0.dist-info/WHEEL +4 -0
  136. strop_cli-0.1.0.dist-info/entry_points.txt +2 -0
  137. strop_cli-0.1.0.dist-info/licenses/LICENSE +21 -0
@@ -0,0 +1,157 @@
1
+ """Rendering the report dictionary as a Markdown document."""
2
+
3
+ from typing import Any
4
+
5
+ from strop.formatting import fmt
6
+
7
+
8
+ def render_markdown(report: dict[str, Any]) -> str:
9
+ """A readable report. Everything here comes from the stored results; nothing is estimated."""
10
+ sections = [
11
+ _title(report),
12
+ _results(report),
13
+ _changes(report),
14
+ _history(report),
15
+ _runs(report),
16
+ _errors(report),
17
+ _usage(report),
18
+ _limitations(report),
19
+ ]
20
+ return "\n\n".join(s for s in sections if s) + "\n"
21
+
22
+
23
+ def _title(report: dict[str, Any]) -> str:
24
+ project, prompt = report["project"], report["prompt"]
25
+ lines = [
26
+ f"# Strop report: {project['name']}",
27
+ "",
28
+ f"Task: {project['task']}",
29
+ "",
30
+ f"Selected prompt: v{prompt['selected']} (initial v{prompt['initial']}, "
31
+ f"best v{prompt['best']}). Generated {report['generated_at']} by Strop "
32
+ f"{report['strop_version']}.",
33
+ ]
34
+ models = report["models"]
35
+ lines.append("")
36
+ lines.append(
37
+ "Models: " + ", ".join(f"{role} `{name}`" for role, name in models.items() if name)
38
+ )
39
+ return "\n".join(lines)
40
+
41
+
42
+ def _results(report: dict[str, Any]) -> str:
43
+ primary = report["metrics"]["primary"]
44
+ rows = []
45
+ for label, key in (
46
+ ("Initial prompt", "initial_validation"),
47
+ ("Selected prompt", "selected_validation"),
48
+ ("Initial prompt", "initial_test"),
49
+ ("Selected prompt", "selected_test"),
50
+ ):
51
+ record = report["results"][key]
52
+ if record is None:
53
+ continue
54
+ value = record["metrics"][primary]["value"]
55
+ rows.append(
56
+ f"| {label} (v{record['version'].lstrip('v')}) | {record['partition']} | "
57
+ f"{fmt(value)} | {record['samples']} |"
58
+ )
59
+ if not rows:
60
+ return "## Results\n\nNo evaluations are stored yet."
61
+ header = f"| Prompt | Partition | {primary} | Examples |\n| --- | --- | --- | --- |"
62
+ return "## Results\n\n" + header + "\n" + "\n".join(rows)
63
+
64
+
65
+ def _changes(report: dict[str, Any]) -> str:
66
+ diff = report["diff"]
67
+ if not diff["changed"]:
68
+ return "## Prompt changes\n\nThe selected prompt is the initial prompt."
69
+ return "## Prompt changes\n\n```diff\n" + _diff_lines(diff) + "\n```"
70
+
71
+
72
+ def _diff_lines(diff: dict[str, Any]) -> str:
73
+ out = []
74
+ for message in diff["messages"]:
75
+ out.append(f"# message {message['index']} ({message['role']}): {message['status']}")
76
+ for line in message["lines"]:
77
+ prefix = {"add": "+", "remove": "-", "context": " ", "hunk": ""}[line["kind"]]
78
+ out.append(f"{prefix}{line['text']}")
79
+ return "\n".join(out)
80
+
81
+
82
+ def _history(report: dict[str, Any]) -> str:
83
+ lines = [
84
+ "## Version history",
85
+ "",
86
+ "| Version | Parent | Source | Decision | Change | Summary |",
87
+ "| --- | --- | --- | --- | --- | --- |",
88
+ ]
89
+ for v in report["versions"]:
90
+ delta = "" if v["primary_delta"] is None else f"{v['primary_delta']:+.4f}"
91
+ parent = f"v{v['parent']}" if v["parent"] else ""
92
+ lines.append(
93
+ f"| v{v['version']} | {parent} | {v['source']} | {v['status'] or ''} | {delta} | "
94
+ f"{_cell(v['change_summary'])} |"
95
+ )
96
+ return "\n".join(lines)
97
+
98
+
99
+ def _runs(report: dict[str, Any]) -> str:
100
+ if not report["runs"]:
101
+ return ""
102
+ lines = [
103
+ "## Optimization runs",
104
+ "",
105
+ "| Run | Status | Stopped because | Turns | Model calls |",
106
+ "| --- | --- | --- | --- | --- |",
107
+ ]
108
+ lines += [
109
+ f"| {r['id']} | {r['status']} | {r['stop_reason'] or ''} | {r['turns']} | "
110
+ f"{r['model_calls']} |"
111
+ for r in report["runs"]
112
+ ]
113
+ return "\n".join(lines)
114
+
115
+
116
+ def _errors(report: dict[str, Any]) -> str:
117
+ analysis = report["error_analysis"]
118
+ if analysis is None:
119
+ return ""
120
+ lines = [
121
+ "## Errors on validation",
122
+ "",
123
+ f"{analysis['failed']} of {analysis['examples']} examples failed (counts only).",
124
+ ]
125
+ lines += [f"- {code}: {count}" for code, count in sorted(analysis["codes"].items())]
126
+ per_class = report["per_class"]
127
+ if per_class:
128
+ lines += [
129
+ "",
130
+ "| Class | Precision | Recall | F1 | Support |",
131
+ "| --- | --- | --- | --- | --- |",
132
+ ]
133
+ lines += [
134
+ f"| {name} | {fmt(s['precision'])} | {fmt(s['recall'])} | {fmt(s['f1'])} | "
135
+ f"{s['support']} |"
136
+ for name, s in per_class.items()
137
+ ]
138
+ return "\n".join(lines)
139
+
140
+
141
+ def _usage(report: dict[str, Any]) -> str:
142
+ usage = report["usage"]
143
+ cost = "unavailable" if usage["cost_usd"] is None else f"${usage['cost_usd']:.4f}"
144
+ tokens = "unavailable" if usage["total_tokens"] is None else f"{usage['total_tokens']:,}"
145
+ return (
146
+ "## Usage\n\n"
147
+ f"{usage['evaluations']} evaluations, {usage['model_calls']} model calls, "
148
+ f"tokens {tokens}, cost {cost}."
149
+ )
150
+
151
+
152
+ def _limitations(report: dict[str, Any]) -> str:
153
+ return "## Limitations\n\n" + "\n".join(f"- {note}" for note in report["limitations"])
154
+
155
+
156
+ def _cell(text: str) -> str:
157
+ return text.replace("|", "/").replace("\n", " ")
@@ -0,0 +1,118 @@
1
+ """Terminal rendering and JSON payloads for optimization runs."""
2
+
3
+ from typing import Any
4
+
5
+ from rich.console import Console
6
+ from rich.table import Table
7
+ from rich.text import Text
8
+
9
+ from strop.formatting import fmt, shorten
10
+ from strop.prompts.diff import diff_templates
11
+ from strop.reporting.diff_view import render_diff
12
+ from strop.services.optimize import RunSummary
13
+ from strop.storage.runs import CandidateRecord
14
+
15
+ _STOP_TEXT = {
16
+ "max_turns": "reached the maximum number of turns",
17
+ "target_reached": "reached the target metric",
18
+ "patience": "no accepted improvement for the configured number of turns",
19
+ "no_failures": "no failures left on the training examples",
20
+ "budget": "stopped by a budget",
21
+ "awaiting_approval": "paused for approval",
22
+ }
23
+
24
+
25
+ def stop_text(summary: RunSummary) -> str:
26
+ text = _STOP_TEXT.get(summary.stop_reason, summary.stop_reason)
27
+ if summary.stop_detail:
28
+ text += f" ({summary.stop_detail})"
29
+ if summary.pending is not None:
30
+ text += (
31
+ f": v{summary.pending.version_number} is waiting for 'strop approve' or 'strop reject'"
32
+ )
33
+ return text
34
+
35
+
36
+ def render_run(console: Console, summary: RunSummary) -> None:
37
+ """Print the run result: why it stopped, the metric change, candidates, and the prompt diff."""
38
+ console.print(
39
+ Text(f"Run {summary.run.id}: {stop_text(summary)}.", style="bold"), soft_wrap=True
40
+ )
41
+ primary = _primary(summary)
42
+ before, after = _value(summary.baseline), _value(summary.result)
43
+ change = (
44
+ f"{summary.start.ref} {primary} {fmt(before)} -> "
45
+ f"{summary.final.ref} {primary} {fmt(after)} (validation)"
46
+ )
47
+ console.print(change, soft_wrap=True, highlight=False)
48
+ if summary.candidates:
49
+ console.print(_candidate_table(summary.candidates))
50
+ cost = "unavailable" if summary.cost_usd is None else f"${summary.cost_usd:.4f}"
51
+ console.print(Text(f"Model calls: {summary.calls} | Cost: {cost}", style="dim"), soft_wrap=True)
52
+ for warning in summary.warnings:
53
+ console.print(f"Warning: {warning}", soft_wrap=True, markup=False, highlight=False)
54
+ _render_final_diff(console, summary)
55
+
56
+
57
+ def _render_final_diff(console: Console, summary: RunSummary) -> None:
58
+ if summary.final.number == summary.initial.number:
59
+ console.print("No improvement was accepted; the best prompt is unchanged.", soft_wrap=True)
60
+ return
61
+ diff = diff_templates(summary.initial.template, summary.final.template)
62
+ console.print(
63
+ Text(f"Prompt changes, {summary.initial.ref} -> {summary.final.ref}:", style="bold"),
64
+ soft_wrap=True,
65
+ )
66
+ render_diff(diff, console)
67
+
68
+
69
+ def _candidate_table(candidates: list[CandidateRecord]) -> Table:
70
+ table = Table(box=None, pad_edge=False, header_style="bold")
71
+ for column in ("Turn", "Version", "Result", "Change", "Summary"):
72
+ table.add_column(column)
73
+ for item in candidates:
74
+ delta = "-" if item.primary_delta is None else f"{item.primary_delta:+.4f}"
75
+ version = f"v{item.version_number}" if item.version_number else "-"
76
+ note = item.change_summary or item.reason
77
+ table.add_row(str(item.turn), version, item.status, delta, shorten(note, 60))
78
+ return table
79
+
80
+
81
+ def _primary(summary: RunSummary) -> str:
82
+ record = summary.result or summary.baseline
83
+ return record.report.primary if record else "primary metric"
84
+
85
+
86
+ def _value(record: Any) -> float | None:
87
+ return record.report.primary_value.value if record is not None else None
88
+
89
+
90
+ def run_payload(summary: RunSummary) -> dict[str, Any]:
91
+ return {
92
+ "run": summary.run.id,
93
+ "status": summary.run.status,
94
+ "stop_reason": summary.stop_reason,
95
+ "stop_detail": summary.stop_detail,
96
+ "initial_version": summary.initial.number,
97
+ "start_version": summary.start.number,
98
+ "best_version": summary.final.number,
99
+ "primary_metric": _primary(summary),
100
+ "start_value": _value(summary.baseline),
101
+ "best_value": _value(summary.result),
102
+ "turns": len([t for t in summary.turns if t.status == "done"]),
103
+ "pending_version": summary.pending.version_number if summary.pending else None,
104
+ "model_calls": summary.calls,
105
+ "cost_usd": summary.cost_usd,
106
+ "candidates": [
107
+ {
108
+ "turn": c.turn,
109
+ "version": c.version_number,
110
+ "status": c.status,
111
+ "primary_delta": c.primary_delta,
112
+ "summary": c.change_summary,
113
+ "reason": c.reason,
114
+ }
115
+ for c in summary.candidates
116
+ ],
117
+ "diff": diff_templates(summary.initial.template, summary.final.template).to_payload(),
118
+ }
@@ -0,0 +1 @@
1
+ """Application workflows used by the command-line layer."""
@@ -0,0 +1,124 @@
1
+ """Analyzing the failures of one prompt version on the train partition."""
2
+
3
+ import sqlite3
4
+ from dataclasses import dataclass
5
+ from typing import Any
6
+
7
+ from strop.analysis import (
8
+ AnalysisFacts,
9
+ FailedExample,
10
+ Pattern,
11
+ build_facts,
12
+ find_patterns,
13
+ select_evidence,
14
+ )
15
+ from strop.config.enums import ModelRole
16
+ from strop.config.llm import ModelSpec
17
+ from strop.config.optimization import OptimizationConfig
18
+ from strop.config.overrides import apply_overrides
19
+ from strop.data.models import Partition
20
+ from strop.errors import InvalidModelReplyError
21
+ from strop.evaluation.budget import BudgetTracker
22
+ from strop.evaluation.budgeted import BudgetedClient
23
+ from strop.evaluation.models import EvaluationRecord
24
+ from strop.llm.client import LLMClient
25
+ from strop.llm.factory import build_client
26
+ from strop.prompts.versions import PromptVersion
27
+ from strop.services.evaluate import sample_rows
28
+ from strop.services.evaluator import Evaluator, RunTag, build_evaluator
29
+ from strop.services.project import Project
30
+ from strop.services.prompts import ensure_initial_version, resolve_version
31
+ from strop.storage.cache import ResponseCache
32
+ from strop.storage.prompts import PromptRepository
33
+
34
+
35
+ @dataclass(frozen=True)
36
+ class TurnAnalysis:
37
+ train_record: EvaluationRecord
38
+ facts: AnalysisFacts
39
+ evidence: tuple[FailedExample, ...]
40
+ patterns: tuple[Pattern, ...]
41
+
42
+ def to_json(self) -> dict[str, Any]:
43
+ """The analysis as stored with a turn. Patterns are labeled as hypotheses."""
44
+ return {
45
+ "facts": {
46
+ "examples": self.facts.total,
47
+ "failed": self.facts.failed,
48
+ "codes": dict(self.facts.codes),
49
+ "confusions": [
50
+ {"expected": e, "got": g, "count": n} for e, g, n in self.facts.confusions
51
+ ],
52
+ },
53
+ "hypotheses": [
54
+ {"name": p.name, "description": p.description, "examples": list(p.example_ids)}
55
+ for p in self.patterns
56
+ ],
57
+ }
58
+
59
+
60
+ async def analyze_version(
61
+ evaluator: Evaluator,
62
+ version: PromptVersion,
63
+ settings: OptimizationConfig,
64
+ *,
65
+ seed: int,
66
+ analyzer: tuple[LLMClient, ModelSpec] | None,
67
+ run_id: int | None = None,
68
+ ) -> TurnAnalysis:
69
+ """Evaluate ``version`` on train, then collect facts, evidence, and (optionally) patterns."""
70
+ rows = evaluator.rows(Partition.TRAIN)
71
+ sampled = settings.train_sample is not None and settings.train_sample < len(rows)
72
+ if sampled and settings.train_sample is not None:
73
+ rows = sample_rows(rows, settings.train_sample, seed)
74
+ record = None if sampled else evaluator.reusable(version, Partition.TRAIN)
75
+ if record is None:
76
+ tag = RunTag("analysis", {"sampled": sampled}, run_id)
77
+ record = await evaluator.evaluate(version, Partition.TRAIN, tag, rows=rows)
78
+ outcomes = evaluator.repository.outcomes(record.id)
79
+ config = evaluator.project.config
80
+ facts = build_facts(outcomes, rows, config.output.type, evaluator.suite.primary)
81
+ evidence = select_evidence(facts.failures, settings.evidence_examples, seed)
82
+ patterns: list[Pattern] = []
83
+ if analyzer is not None and evidence:
84
+ client, spec = analyzer
85
+ try:
86
+ patterns = await find_patterns(
87
+ client,
88
+ spec,
89
+ task=config.project.task_description,
90
+ output=config.output,
91
+ facts=facts,
92
+ evidence=evidence,
93
+ max_chars=settings.evidence_chars,
94
+ )
95
+ except InvalidModelReplyError:
96
+ patterns = []
97
+ return TurnAnalysis(record, facts, tuple(evidence), tuple(patterns))
98
+
99
+
100
+ async def run_analysis(
101
+ project: Project,
102
+ conn: sqlite3.Connection,
103
+ ref: str | None,
104
+ *,
105
+ patterns: bool,
106
+ examples: int | None,
107
+ use_cache: bool,
108
+ ) -> tuple[PromptVersion, TurnAnalysis]:
109
+ """Analyze the train failures of a prompt version, optionally asking for patterns."""
110
+ repo = PromptRepository(conn)
111
+ ensure_initial_version(project, repo)
112
+ version = resolve_version(repo, ref)
113
+ settings = apply_overrides(project.config.optimization, {"evidence_examples": examples})
114
+ budget = BudgetTracker(project.config.budgets)
115
+ evaluator = build_evaluator(project, conn, use_cache=use_cache, budget=budget)
116
+ analyzer = None
117
+ if patterns:
118
+ cache = ResponseCache(conn) if use_cache else None
119
+ client = BudgetedClient(build_client(project.config.execution, cache), budget)
120
+ analyzer = (client, project.config.models.resolve(ModelRole.ANALYZER))
121
+ analysis = await analyze_version(
122
+ evaluator, version, settings, seed=evaluator.partitions.seed, analyzer=analyzer
123
+ )
124
+ return version, analysis
@@ -0,0 +1,38 @@
1
+ """Deleting stored data on request."""
2
+
3
+ import shutil
4
+
5
+ from strop.errors import ConfigError
6
+ from strop.services.project import Project
7
+ from strop.storage.cache import ResponseCache
8
+
9
+ CHOICES = "--cache, --outputs, or --all"
10
+
11
+
12
+ def clean_project(
13
+ project: Project, *, cache: bool, outputs: bool, everything: bool, confirmed: bool
14
+ ) -> list[str]:
15
+ """Delete the chosen data and report what was removed; nothing goes without ``confirmed``."""
16
+ if not (cache or outputs or everything):
17
+ raise ConfigError(f"choose what to delete: {CHOICES}")
18
+ if not confirmed:
19
+ raise ConfigError("this permanently deletes stored data; pass --yes to confirm")
20
+ messages = []
21
+ if everything:
22
+ shutil.rmtree(project.paths.state_dir, ignore_errors=True)
23
+ return [
24
+ f"Deleted {project.paths.state_dir.name}/ (database, cache, prompt versions, results)",
25
+ "strop.yaml, the prompt file, and the dataset were not touched.",
26
+ ]
27
+ with project.database() as conn:
28
+ if cache or outputs:
29
+ removed = ResponseCache(conn).clear()
30
+ messages.append(f"Deleted {removed} cached model response(s).")
31
+ if outputs:
32
+ cleared = conn.execute(
33
+ "UPDATE example_results SET raw_output = '', value = NULL, parsed_json = NULL, "
34
+ "detail = NULL, feedback = NULL"
35
+ ).rowcount
36
+ conn.execute("UPDATE candidates SET raw_response = ''")
37
+ messages.append(f"Cleared the stored model output of {cleared} example result(s).")
38
+ return messages
@@ -0,0 +1,60 @@
1
+ """Running one prompt version on several target models."""
2
+
3
+ import sqlite3
4
+ from dataclasses import dataclass, replace
5
+
6
+ from strop.data.models import Partition
7
+ from strop.errors import ConfigError
8
+ from strop.evaluation.models import EvaluationRecord
9
+ from strop.evaluation.runner import Progress
10
+ from strop.prompts.versions import PromptVersion
11
+ from strop.services.evaluate import sample_rows
12
+ from strop.services.evaluator import RunTag, build_evaluator
13
+ from strop.services.project import Project
14
+ from strop.services.prompts import ensure_initial_version, resolve_version
15
+ from strop.storage.prompts import PromptRepository
16
+
17
+
18
+ @dataclass(frozen=True)
19
+ class Comparison:
20
+ version: PromptVersion
21
+ partition: Partition
22
+ results: list[tuple[str, EvaluationRecord]]
23
+
24
+
25
+ def parse_models(text: str) -> list[str]:
26
+ """Split a comma-separated list of model names, rejecting empty and repeated entries."""
27
+ names = [item.strip() for item in text.split(",") if item.strip()]
28
+ if len(names) < 2:
29
+ raise ConfigError("--models needs at least two comma-separated model names")
30
+ if len(set(names)) != len(names):
31
+ raise ConfigError("--models lists the same model more than once")
32
+ return names
33
+
34
+
35
+ async def run_compare( # noqa: PLR0913
36
+ project: Project,
37
+ conn: sqlite3.Connection,
38
+ *,
39
+ models: list[str],
40
+ ref: str | None,
41
+ partition: Partition,
42
+ sample: int | None,
43
+ use_cache: bool,
44
+ progress: Progress | None = None,
45
+ ) -> Comparison:
46
+ """Evaluate the same version on the same rows with each model as the target."""
47
+ repo = PromptRepository(conn)
48
+ ensure_initial_version(project, repo)
49
+ version = resolve_version(repo, ref)
50
+ base = build_evaluator(project, conn, use_cache=use_cache)
51
+ rows = base.rows(partition)
52
+ if sample is not None:
53
+ rows = sample_rows(rows, sample, base.partitions.seed)
54
+ results = []
55
+ for name in models:
56
+ evaluator = replace(base, target=base.target.model_copy(update={"model": name}))
57
+ tag = RunTag("compare", {"sampled": sample is not None})
58
+ record = await evaluator.evaluate(version, partition, tag, rows=rows, progress=progress)
59
+ results.append((name, record))
60
+ return Comparison(version, partition, results)
@@ -0,0 +1,70 @@
1
+ """Everything the optimization loop needs, assembled once per command."""
2
+
3
+ import sqlite3
4
+ from collections.abc import Callable
5
+ from dataclasses import dataclass
6
+
7
+ from strop.config.enums import ModelRole
8
+ from strop.config.limits import BudgetsConfig
9
+ from strop.config.llm import ModelSpec
10
+ from strop.config.optimization import OptimizationConfig
11
+ from strop.evaluation.budget import BudgetTracker
12
+ from strop.evaluation.budgeted import BudgetedClient
13
+ from strop.llm.client import LLMClient
14
+ from strop.llm.factory import build_client
15
+ from strop.optimization import AcceptanceRules, CandidateGenerator
16
+ from strop.services.evaluator import Evaluator, build_evaluator
17
+ from strop.services.project import Project
18
+ from strop.storage.cache import ResponseCache
19
+ from strop.storage.prompts import PromptRepository
20
+ from strop.storage.runs import RunRepository
21
+
22
+ Say = Callable[[str], None]
23
+
24
+
25
+ @dataclass(frozen=True)
26
+ class LoopContext:
27
+ project: Project
28
+ settings: OptimizationConfig
29
+ evaluator: Evaluator
30
+ prompts: PromptRepository
31
+ runs: RunRepository
32
+ generator: CandidateGenerator
33
+ analyzer_client: LLMClient
34
+ analyzer_spec: ModelSpec
35
+ rules: AcceptanceRules
36
+ say: Say
37
+
38
+ @property
39
+ def budget(self) -> BudgetTracker:
40
+ return self.evaluator.budget
41
+
42
+
43
+ def build_context(
44
+ project: Project,
45
+ conn: sqlite3.Connection,
46
+ settings: OptimizationConfig,
47
+ budgets: BudgetsConfig,
48
+ *,
49
+ use_cache: bool,
50
+ say: Say,
51
+ ) -> LoopContext:
52
+ """Wire the evaluator, optimizer, analyzer and acceptance rules to one shared budget."""
53
+ config = project.config
54
+ budget = BudgetTracker(budgets)
55
+ evaluator = build_evaluator(project, conn, use_cache=use_cache, budget=budget)
56
+ cache = ResponseCache(conn) if use_cache else None
57
+ helper = BudgetedClient(build_client(config.execution, cache), budget)
58
+ optimizer = config.models.resolve(ModelRole.OPTIMIZER)
59
+ return LoopContext(
60
+ project=project,
61
+ settings=settings,
62
+ evaluator=evaluator,
63
+ prompts=PromptRepository(conn),
64
+ runs=RunRepository(conn),
65
+ generator=CandidateGenerator(helper, optimizer),
66
+ analyzer_client=helper,
67
+ analyzer_spec=config.models.resolve(ModelRole.ANALYZER),
68
+ rules=AcceptanceRules.from_config(settings, config.metrics.constraints),
69
+ say=say,
70
+ )
strop/services/data.py ADDED
@@ -0,0 +1,65 @@
1
+ """Dataset workflows: validation and creating persisted splits."""
2
+
3
+ from dataclasses import dataclass
4
+ from typing import Any
5
+
6
+ from strop.config.overrides import apply_overrides
7
+ from strop.config.split import SplitConfig
8
+ from strop.data import Dataset, SplitSummary, load_dataset, require_valid, split_rows, summarize
9
+ from strop.data.splitting import stratify_groups
10
+ from strop.services.project import Project
11
+ from strop.storage.datasets import DatasetRepository
12
+
13
+
14
+ @dataclass(frozen=True)
15
+ class SplitOutcome:
16
+ split_id: int
17
+ created: bool
18
+ dataset: Dataset
19
+ config: SplitConfig
20
+ summary: SplitSummary
21
+
22
+
23
+ def check_dataset(project: Project) -> Dataset:
24
+ """Load and validate the dataset without raising on invalid rows."""
25
+ return load_dataset(project.config, project.root)
26
+
27
+
28
+ def effective_split_config(project: Project, overrides: dict[str, Any]) -> SplitConfig:
29
+ """The configured split settings with any non-None command-line overrides applied."""
30
+ return apply_overrides(project.config.split, overrides)
31
+
32
+
33
+ def create_split(project: Project, split: SplitConfig, *, skip_invalid: bool) -> SplitOutcome:
34
+ """Split the dataset and persist it. An identical existing split is reused, not recomputed."""
35
+ dataset = require_valid(check_dataset(project), skip_invalid=skip_invalid)
36
+ conn = project.connect()
37
+ try:
38
+ repo = DatasetRepository(conn)
39
+ dataset_id = repo.save_dataset(
40
+ content_hash=dataset.content_hash,
41
+ source_path=str(project.config.dataset.path),
42
+ row_count=len(dataset.rows),
43
+ skipped_rows=dataset.total_records - len(dataset.rows),
44
+ skip_invalid=skip_invalid,
45
+ )
46
+ config_json = split.model_dump_json()
47
+ split_id = repo.find_split(dataset_id, config_json)
48
+ created = split_id is None
49
+ if split_id is None:
50
+ assignments = split_rows(dataset.rows, split)
51
+ stratified = len(stratify_groups(dataset.rows, split)) > 1
52
+ strategy = "stratified" if stratified else "random"
53
+ split_id = repo.create_split(
54
+ dataset_id=dataset_id,
55
+ seed=split.seed,
56
+ strategy=strategy,
57
+ config_json=config_json,
58
+ assignments=assignments,
59
+ )
60
+ assignments = repo.assignments(split_id)
61
+ finally:
62
+ conn.close()
63
+ return SplitOutcome(
64
+ split_id, created, dataset, split, summarize(dataset.rows, assignments, split)
65
+ )