strop-cli 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (137) hide show
  1. strop/__init__.py +5 -0
  2. strop/__main__.py +5 -0
  3. strop/analysis/__init__.py +17 -0
  4. strop/analysis/classify.py +52 -0
  5. strop/analysis/evidence.py +52 -0
  6. strop/analysis/facts.py +74 -0
  7. strop/analysis/patterns.py +90 -0
  8. strop/cli/__init__.py +5 -0
  9. strop/cli/analyze.py +93 -0
  10. strop/cli/app.py +82 -0
  11. strop/cli/data.py +111 -0
  12. strop/cli/evaluate.py +145 -0
  13. strop/cli/export.py +74 -0
  14. strop/cli/final.py +106 -0
  15. strop/cli/init.py +131 -0
  16. strop/cli/optimize.py +156 -0
  17. strop/cli/options.py +20 -0
  18. strop/cli/progress.py +18 -0
  19. strop/cli/prompts.py +110 -0
  20. strop/cli/status.py +68 -0
  21. strop/cli/versions.py +123 -0
  22. strop/config/__init__.py +16 -0
  23. strop/config/base.py +9 -0
  24. strop/config/constraints.py +33 -0
  25. strop/config/dataset.py +63 -0
  26. strop/config/enums.py +52 -0
  27. strop/config/io.py +52 -0
  28. strop/config/limits.py +19 -0
  29. strop/config/llm.py +42 -0
  30. strop/config/metrics.py +86 -0
  31. strop/config/optimization.py +23 -0
  32. strop/config/output.py +31 -0
  33. strop/config/overrides.py +22 -0
  34. strop/config/root.py +36 -0
  35. strop/config/rules.py +67 -0
  36. strop/config/split.py +23 -0
  37. strop/data/__init__.py +58 -0
  38. strop/data/loaders.py +85 -0
  39. strop/data/models.py +44 -0
  40. strop/data/rows.py +99 -0
  41. strop/data/splitting.py +103 -0
  42. strop/env.py +44 -0
  43. strop/errors.py +59 -0
  44. strop/evaluation/__init__.py +1 -0
  45. strop/evaluation/budget.py +67 -0
  46. strop/evaluation/budgeted.py +23 -0
  47. strop/evaluation/examples.py +49 -0
  48. strop/evaluation/models.py +105 -0
  49. strop/evaluation/parsing.py +96 -0
  50. strop/evaluation/runner.py +107 -0
  51. strop/formatting.py +14 -0
  52. strop/llm/__init__.py +7 -0
  53. strop/llm/caching.py +28 -0
  54. strop/llm/client.py +15 -0
  55. strop/llm/factory.py +14 -0
  56. strop/llm/litellm_client.py +90 -0
  57. strop/llm/mock.py +43 -0
  58. strop/llm/retry.py +44 -0
  59. strop/llm/structured.py +52 -0
  60. strop/llm/types.py +59 -0
  61. strop/log.py +49 -0
  62. strop/metrics/__init__.py +1 -0
  63. strop/metrics/basic.py +30 -0
  64. strop/metrics/classification.py +164 -0
  65. strop/metrics/judge.py +130 -0
  66. strop/metrics/python_metric.py +101 -0
  67. strop/metrics/python_runner.py +65 -0
  68. strop/metrics/registry.py +115 -0
  69. strop/metrics/rules.py +55 -0
  70. strop/metrics/structured.py +109 -0
  71. strop/metrics/suite.py +109 -0
  72. strop/metrics/text.py +55 -0
  73. strop/metrics/types.py +106 -0
  74. strop/optimization/__init__.py +25 -0
  75. strop/optimization/acceptance.py +170 -0
  76. strop/optimization/generator.py +92 -0
  77. strop/optimization/meta_prompt.py +101 -0
  78. strop/optimization/types.py +69 -0
  79. strop/prompts/__init__.py +18 -0
  80. strop/prompts/diff.py +139 -0
  81. strop/prompts/files.py +92 -0
  82. strop/prompts/models.py +51 -0
  83. strop/prompts/render.py +16 -0
  84. strop/prompts/template.py +45 -0
  85. strop/prompts/validation.py +31 -0
  86. strop/prompts/versions.py +34 -0
  87. strop/py.typed +0 -0
  88. strop/reporting/__init__.py +6 -0
  89. strop/reporting/console.py +20 -0
  90. strop/reporting/diff_view.py +102 -0
  91. strop/reporting/evaluation_view.py +141 -0
  92. strop/reporting/final_view.py +109 -0
  93. strop/reporting/prompt_export.py +58 -0
  94. strop/reporting/report_markdown.py +157 -0
  95. strop/reporting/run_view.py +118 -0
  96. strop/services/__init__.py +1 -0
  97. strop/services/analysis.py +124 -0
  98. strop/services/clean.py +38 -0
  99. strop/services/compare.py +60 -0
  100. strop/services/context.py +70 -0
  101. strop/services/data.py +65 -0
  102. strop/services/evaluate.py +105 -0
  103. strop/services/evaluator.py +169 -0
  104. strop/services/export.py +90 -0
  105. strop/services/final_test.py +69 -0
  106. strop/services/metrics.py +89 -0
  107. strop/services/optimize.py +187 -0
  108. strop/services/partitions.py +46 -0
  109. strop/services/project.py +55 -0
  110. strop/services/prompts.py +76 -0
  111. strop/services/report.py +194 -0
  112. strop/services/review.py +43 -0
  113. strop/services/scaffold.py +79 -0
  114. strop/services/turn.py +234 -0
  115. strop/storage/__init__.py +6 -0
  116. strop/storage/cache.py +45 -0
  117. strop/storage/database.py +45 -0
  118. strop/storage/datasets.py +129 -0
  119. strop/storage/evaluations.py +160 -0
  120. strop/storage/migrations/001_initial.sql +5 -0
  121. strop/storage/migrations/002_datasets.sql +27 -0
  122. strop/storage/migrations/003_prompt_versions.sql +22 -0
  123. strop/storage/migrations/004_llm_cache.sql +6 -0
  124. strop/storage/migrations/005_evaluations.sql +37 -0
  125. strop/storage/migrations/006_runs.sql +52 -0
  126. strop/storage/migrations/007_example_feedback.sql +2 -0
  127. strop/storage/migrations/__init__.py +5 -0
  128. strop/storage/migrator.py +91 -0
  129. strop/storage/paths.py +30 -0
  130. strop/storage/prompts.py +96 -0
  131. strop/storage/run_models.py +75 -0
  132. strop/storage/runs.py +263 -0
  133. strop_cli-0.1.0.dist-info/METADATA +242 -0
  134. strop_cli-0.1.0.dist-info/RECORD +137 -0
  135. strop_cli-0.1.0.dist-info/WHEEL +4 -0
  136. strop_cli-0.1.0.dist-info/entry_points.txt +2 -0
  137. strop_cli-0.1.0.dist-info/licenses/LICENSE +21 -0
@@ -0,0 +1,187 @@
1
+ """The optimization loop: run turns until a stop condition is reached."""
2
+
3
+ import asyncio
4
+ import sqlite3
5
+ from dataclasses import dataclass, field
6
+
7
+ from strop.config.limits import BudgetsConfig
8
+ from strop.config.optimization import OptimizationConfig
9
+ from strop.data.models import Partition
10
+ from strop.errors import BudgetExceededError, ConfigError, DataError
11
+ from strop.evaluation.models import EvaluationRecord
12
+ from strop.metrics.types import Direction
13
+ from strop.prompts.versions import PromptVersion
14
+ from strop.services.context import LoopContext, Say, build_context
15
+ from strop.services.project import Project
16
+ from strop.services.prompts import ensure_initial_version
17
+ from strop.services.turn import TurnRunner
18
+ from strop.storage.runs import CandidateRecord, RunRecord, TurnRecord
19
+
20
+
21
+ @dataclass(frozen=True)
22
+ class RunSummary:
23
+ """What an optimization run did and why it stopped."""
24
+
25
+ run: RunRecord
26
+ initial: PromptVersion
27
+ start: PromptVersion
28
+ final: PromptVersion
29
+ baseline: EvaluationRecord | None
30
+ result: EvaluationRecord | None
31
+ turns: list[TurnRecord]
32
+ candidates: list[CandidateRecord]
33
+ stop_reason: str
34
+ stop_detail: str
35
+ pending: CandidateRecord | None
36
+ calls: int
37
+ cost_usd: float | None
38
+ warnings: list[str] = field(default_factory=list)
39
+
40
+
41
+ async def optimize(
42
+ project: Project,
43
+ conn: sqlite3.Connection,
44
+ settings: OptimizationConfig,
45
+ budgets: BudgetsConfig,
46
+ *,
47
+ resume: bool,
48
+ use_cache: bool,
49
+ say: Say,
50
+ ) -> RunSummary:
51
+ """Run (or resume) the loop: analyze failures, propose revisions, test, and decide."""
52
+ ctx = build_context(project, conn, settings, budgets, use_cache=use_cache, say=say)
53
+ validation = len(ctx.evaluator.rows(Partition.VALIDATION))
54
+ if validation < settings.min_validation_samples:
55
+ raise DataError(
56
+ f"the validation partition has {validation} rows, fewer than the "
57
+ f"{settings.min_validation_samples} needed to compare prompts; add data"
58
+ )
59
+ run = _begin(ctx, resume)
60
+ runner = TurnRunner(ctx, run)
61
+ reason, detail = "", ""
62
+ try:
63
+ reason = await _turns(ctx, run, runner)
64
+ except BudgetExceededError as exc:
65
+ reason, detail = "budget", str(exc)
66
+ except (KeyboardInterrupt, asyncio.CancelledError):
67
+ _save(ctx, run.id, status="interrupted")
68
+ raise
69
+ except BaseException as exc:
70
+ _save(ctx, run.id, status="failed", stop_reason=str(exc)[:500])
71
+ raise
72
+ status = {"awaiting_approval": "paused", "budget": "interrupted"}.get(reason, "completed")
73
+ _save(ctx, run.id, status=status, stop_reason=reason, finished=status == "completed")
74
+ return _summary(ctx, run.id, reason, detail)
75
+
76
+
77
+ def _begin(ctx: LoopContext, resume: bool) -> RunRecord:
78
+ pending = ctx.runs.pending()
79
+ if pending:
80
+ ref = f"v{pending[0].version_number}"
81
+ raise ConfigError(
82
+ f"{ref} is waiting for a decision; run 'strop approve {ref}' or "
83
+ f"'strop reject {ref}' first"
84
+ )
85
+ if resume:
86
+ run = ctx.runs.resumable_run()
87
+ if run is None:
88
+ raise ConfigError(
89
+ "there is no unfinished run to resume; start one with 'strop optimize'"
90
+ )
91
+ ctx.runs.update_run(run.id, status="running")
92
+ return ctx.runs.get_run(run.id)
93
+ unfinished = ctx.runs.resumable_run()
94
+ if unfinished is not None:
95
+ ctx.runs.update_run(unfinished.id, status="interrupted")
96
+ best = ctx.prompts.best() or ensure_initial_version(ctx.project, ctx.prompts)
97
+ return ctx.runs.create_run(best.number, ctx.settings.model_dump(mode="json"))
98
+
99
+
100
+ async def _turns(ctx: LoopContext, run: RunRecord, runner: TurnRunner) -> str:
101
+ stale = run.stale_turns
102
+ while True:
103
+ turns = ctx.runs.turns(run.id)
104
+ if sum(t.status == "done" for t in turns) >= ctx.settings.max_turns:
105
+ return "max_turns"
106
+ best = _best(ctx)
107
+ record = await runner.validation_record(best)
108
+ if _target_reached(ctx, record):
109
+ return "target_reached"
110
+ open_turn = next((t for t in turns if t.status != "done"), None)
111
+ number = open_turn.number if open_turn else (turns[-1].number + 1 if turns else 1)
112
+ result = await runner.run(number, best)
113
+ if result.awaiting is not None:
114
+ return "awaiting_approval"
115
+ stale = 0 if result.improved else stale + 1
116
+ _save(ctx, run.id, stale_turns=stale)
117
+ if result.no_failures:
118
+ return "no_failures"
119
+ if stale >= ctx.settings.patience:
120
+ return "patience"
121
+
122
+
123
+ def _best(ctx: LoopContext) -> PromptVersion:
124
+ best = ctx.prompts.best()
125
+ if best is None:
126
+ raise ConfigError("no best prompt is set; run 'strop baseline' first")
127
+ return best
128
+
129
+
130
+ def _target_reached(ctx: LoopContext, record: EvaluationRecord) -> bool:
131
+ target = ctx.settings.target
132
+ value = record.report.primary_value.value
133
+ if target is None or value is None:
134
+ return False
135
+ if ctx.evaluator.suite.direction(record.report.primary) is Direction.MAXIMIZE:
136
+ return value >= target
137
+ return value <= target
138
+
139
+
140
+ def _cost(ctx: LoopContext) -> float | None:
141
+ """The known cost so far, or None when no call reported one."""
142
+ budget = ctx.budget
143
+ return budget.cost_usd if budget.calls > budget.cost_unknown_calls else None
144
+
145
+
146
+ def _save(
147
+ ctx: LoopContext,
148
+ run_id: int,
149
+ *,
150
+ status: str | None = None,
151
+ stale_turns: int | None = None,
152
+ stop_reason: str | None = None,
153
+ finished: bool = False,
154
+ ) -> None:
155
+ """Update a run and record the calls and cost used so far."""
156
+ ctx.runs.update_run(
157
+ run_id,
158
+ status=status,
159
+ stale_turns=stale_turns,
160
+ stop_reason=stop_reason,
161
+ model_calls=ctx.budget.calls,
162
+ cost_usd=_cost(ctx),
163
+ finished=finished,
164
+ )
165
+
166
+
167
+ def _summary(ctx: LoopContext, run_id: int, reason: str, detail: str) -> RunSummary:
168
+ run = ctx.runs.get_run(run_id)
169
+ start = ctx.prompts.get(run.start_prompt)
170
+ final = _best(ctx)
171
+ pending = next(iter(ctx.runs.pending()), None)
172
+ return RunSummary(
173
+ run=run,
174
+ initial=ctx.prompts.first() or start,
175
+ start=start,
176
+ final=final,
177
+ baseline=ctx.evaluator.reusable(start, Partition.VALIDATION),
178
+ result=ctx.evaluator.reusable(final, Partition.VALIDATION),
179
+ turns=ctx.runs.turns(run_id),
180
+ candidates=ctx.runs.candidates(run_id),
181
+ stop_reason=reason,
182
+ stop_detail=detail,
183
+ pending=pending,
184
+ calls=ctx.budget.calls,
185
+ cost_usd=_cost(ctx),
186
+ warnings=ctx.budget.warnings(),
187
+ )
@@ -0,0 +1,46 @@
1
+ """Loading the dataset rows of the active split, grouped by partition."""
2
+
3
+ from dataclasses import dataclass, field
4
+
5
+ from strop.data import load_dataset, require_valid
6
+ from strop.data.models import Partition, Row
7
+ from strop.errors import ConfigError, DataError
8
+ from strop.services.project import Project
9
+ from strop.storage.datasets import DatasetRepository
10
+
11
+
12
+ @dataclass(frozen=True)
13
+ class PartitionedData:
14
+ """Rows of the latest split. The test partition is only handed out on explicit request."""
15
+
16
+ split_id: int
17
+ seed: int
18
+ rows: dict[Partition, list[Row]] = field(default_factory=dict)
19
+
20
+ def get(self, partition: Partition) -> list[Row]:
21
+ return self.rows.get(partition, [])
22
+
23
+
24
+ def load_partitions(project: Project, repo: DatasetRepository) -> PartitionedData:
25
+ """Read the dataset and attach the stored split assignments.
26
+
27
+ Fails if there is no split or if the dataset file changed since the split was made.
28
+ """
29
+ split = repo.latest_split()
30
+ if split is None:
31
+ raise ConfigError("no data split yet; run 'strop data split' first")
32
+ dataset = require_valid(
33
+ load_dataset(project.config, project.root), skip_invalid=split.skip_invalid
34
+ )
35
+ if dataset.content_hash != split.content_hash:
36
+ raise DataError(
37
+ "the dataset file changed since the split was created; run 'strop data split' again"
38
+ )
39
+ assignments = repo.assignments(split.id)
40
+ rows: dict[Partition, list[Row]] = {p: [] for p in Partition}
41
+ for row in dataset.rows:
42
+ partition = assignments.get(row.id)
43
+ if partition is None:
44
+ raise DataError(f"row '{row.id}' has no partition; run 'strop data split' again")
45
+ rows[partition].append(row)
46
+ return PartitionedData(split.id, split.seed, rows)
@@ -0,0 +1,55 @@
1
+ """An opened Strop project: its directory, validated configuration, and database."""
2
+
3
+ import sqlite3
4
+ from collections.abc import Iterator
5
+ from contextlib import contextmanager
6
+ from dataclasses import dataclass, replace
7
+ from pathlib import Path
8
+
9
+ from strop.config import StropConfig, load_config
10
+ from strop.config.llm import ModelSpec
11
+ from strop.env import load_env_file
12
+ from strop.storage import ProjectPaths, open_database
13
+
14
+
15
+ @dataclass(frozen=True)
16
+ class Project:
17
+ root: Path
18
+ paths: ProjectPaths
19
+ config: StropConfig
20
+
21
+ def connect(self) -> sqlite3.Connection:
22
+ """Open the project database, creating and migrating it if needed."""
23
+ return open_database(self.paths)
24
+
25
+ def with_models(self, overrides: dict[str, str | None]) -> "Project":
26
+ """A copy that uses the given model names for some roles; None values are ignored."""
27
+ models = self.config.models
28
+ changes: dict[str, ModelSpec] = {}
29
+ for role, name in overrides.items():
30
+ if name:
31
+ current: ModelSpec | None = getattr(models, role)
32
+ changes[role] = (
33
+ current.model_copy(update={"model": name}) if current else ModelSpec(model=name)
34
+ )
35
+ if not changes:
36
+ return self
37
+ config = self.config.model_copy(update={"models": models.model_copy(update=changes)})
38
+ return replace(self, config=config)
39
+
40
+ @contextmanager
41
+ def database(self) -> Iterator[sqlite3.Connection]:
42
+ """Open the project database for the duration of a ``with`` block."""
43
+ conn = self.connect()
44
+ try:
45
+ yield conn
46
+ finally:
47
+ conn.close()
48
+
49
+
50
+ def open_project(root: Path) -> Project:
51
+ """Load strop.yaml from ``root``; raise ``ConfigError`` if it is missing or invalid."""
52
+ paths = ProjectPaths(root)
53
+ config = load_config(paths.config_file)
54
+ load_env_file(root / ".env")
55
+ return Project(root, paths, config)
@@ -0,0 +1,76 @@
1
+ """Prompt version workflows."""
2
+
3
+ from pathlib import Path
4
+
5
+ from strop.errors import ConfigError
6
+ from strop.prompts import PromptTemplate, check_variables, load_prompt_file
7
+ from strop.prompts.versions import PromptVersion, parse_version_ref
8
+ from strop.services.project import Project
9
+ from strop.storage.prompts import PromptRepository
10
+
11
+
12
+ def load_project_prompt(project: Project, path: Path | None = None) -> PromptTemplate:
13
+ """Parse a prompt file and check it only uses variables the dataset provides."""
14
+ template = load_prompt_file(path or project.root / project.config.prompt.file)
15
+ check_variables(template.variables, project.config.dataset.input_columns)
16
+ return template
17
+
18
+
19
+ def ensure_initial_version(project: Project, repo: PromptRepository) -> PromptVersion:
20
+ """Register the prompt file as v1 on first use, and make it the best prompt."""
21
+ existing = repo.first()
22
+ if existing is not None:
23
+ return existing
24
+ template = load_project_prompt(project)
25
+ version = repo.add(
26
+ template,
27
+ parent_number=None,
28
+ change_summary="initial prompt",
29
+ reason=f"registered from {project.config.prompt.file}",
30
+ )
31
+ repo.set_best(version.number)
32
+ return version
33
+
34
+
35
+ def has_unregistered_edits(project: Project, repo: PromptRepository) -> bool:
36
+ """True if the prompt file differs from every stored version."""
37
+ template = load_prompt_file(project.root / project.config.prompt.file)
38
+ return repo.find_by_hash(template.content_hash) is None
39
+
40
+
41
+ def add_version_from_file(
42
+ project: Project,
43
+ repo: PromptRepository,
44
+ file: Path,
45
+ *,
46
+ parent: str | None,
47
+ reason: str,
48
+ use: bool,
49
+ ) -> PromptVersion:
50
+ """Save a manual edit as a new immutable version, optionally making it the best prompt."""
51
+ ensure_initial_version(project, repo)
52
+ template = load_project_prompt(project, file)
53
+ duplicate = repo.find_by_hash(template.content_hash)
54
+ if duplicate is not None:
55
+ raise ConfigError(f"this prompt is identical to the existing version {duplicate.ref}")
56
+ parent_number = resolve_version(repo, parent).number
57
+ version = repo.add(
58
+ template,
59
+ parent_number=parent_number,
60
+ change_summary="manual edit",
61
+ reason=reason,
62
+ source="user",
63
+ )
64
+ if use:
65
+ repo.set_best(version.number)
66
+ return version
67
+
68
+
69
+ def resolve_version(repo: PromptRepository, ref: str | None) -> PromptVersion:
70
+ """The version named by ``ref`` (like ``v3``), or the best version when ``ref`` is None."""
71
+ if ref is not None:
72
+ return repo.get(parse_version_ref(ref))
73
+ best = repo.best()
74
+ if best is None:
75
+ raise ConfigError("no prompt versions yet; run 'strop baseline' or 'strop prompt add'")
76
+ return best
@@ -0,0 +1,194 @@
1
+ """Collecting everything a final report needs from the project database."""
2
+
3
+ import sqlite3
4
+ from datetime import UTC, datetime
5
+ from typing import Any
6
+
7
+ from strop import __version__
8
+ from strop.analysis import build_facts
9
+ from strop.data.models import Partition
10
+ from strop.evaluation.models import EvaluationRecord
11
+ from strop.prompts.diff import diff_templates
12
+ from strop.prompts.versions import PromptVersion
13
+ from strop.reporting.evaluation_view import record_payload
14
+ from strop.services.project import Project
15
+ from strop.storage.datasets import DatasetRepository
16
+ from strop.storage.evaluations import EvaluationRepository
17
+ from strop.storage.prompts import PromptRepository
18
+ from strop.storage.runs import RunRecord, RunRepository
19
+
20
+ SMALL_VALIDATION = 30
21
+
22
+
23
+ def build_report(
24
+ project: Project, conn: sqlite3.Connection, selected: PromptVersion
25
+ ) -> dict[str, Any]:
26
+ """A machine-readable report for ``selected``; missing data is marked unavailable."""
27
+ prompts, evals = PromptRepository(conn), EvaluationRepository(conn)
28
+ datasets, runs = DatasetRepository(conn), RunRepository(conn)
29
+ initial = prompts.first() or selected
30
+ split = datasets.latest_split()
31
+ counts = datasets.partition_counts(split.id) if split else {}
32
+ results = {
33
+ "initial_validation": _latest(evals, initial, Partition.VALIDATION),
34
+ "selected_validation": _latest(evals, selected, Partition.VALIDATION),
35
+ "initial_test": _latest(evals, initial, Partition.TEST),
36
+ "selected_test": _latest(evals, selected, Partition.TEST),
37
+ }
38
+ payload: dict[str, Any] = {
39
+ "generated_at": datetime.now(UTC).isoformat(timespec="seconds"),
40
+ "strop_version": __version__,
41
+ "project": {
42
+ "name": project.config.project.name,
43
+ "task": project.config.project.task_description,
44
+ },
45
+ "dataset": _dataset(datasets, split.dataset_id) if split else None,
46
+ "split": (
47
+ {"id": split.id, "seed": split.seed, "counts": {p.value: n for p, n in counts.items()}}
48
+ if split
49
+ else None
50
+ ),
51
+ "models": _models(project),
52
+ "metrics": _metrics(project),
53
+ "prompt": {
54
+ "selected": selected.number,
55
+ "initial": initial.number,
56
+ "best": prompts.best_number(),
57
+ },
58
+ "results": {
59
+ key: record_payload(rec, f"v{rec.prompt_number}") if rec else None
60
+ for key, rec in results.items()
61
+ },
62
+ "per_class": _per_class(results["selected_validation"]),
63
+ "versions": [_version(v, runs.for_version(v.number)) for v in prompts.list()],
64
+ "runs": [_run(r, runs) for r in runs.all_runs()],
65
+ "diff": diff_templates(initial.template, selected.template).to_payload(),
66
+ "error_analysis": _error_analysis(project, evals, selected),
67
+ "usage": _usage(evals),
68
+ }
69
+ payload["limitations"] = limitations(payload)
70
+ return payload
71
+
72
+
73
+ def limitations(report: dict[str, Any]) -> list[str]:
74
+ """Plain statements about what this report cannot show."""
75
+ notes = []
76
+ validation = (report["results"].get("selected_validation") or {}).get("samples")
77
+ if validation is not None and validation < SMALL_VALIDATION:
78
+ notes.append(
79
+ f"The validation partition has only {validation} examples, so small score "
80
+ "differences may be noise."
81
+ )
82
+ if (report["split"] or {}).get("counts", {}).get("test", 0) == 0:
83
+ notes.append("There is no test partition, so there is no held-out result.")
84
+ elif report["results"].get("selected_test") is None:
85
+ notes.append("The selected prompt has not been evaluated on the test partition yet.")
86
+ if any(m["subjective"] for r in report["results"].values() if r for m in r["metrics"].values()):
87
+ notes.append("LLM-judged metrics are subjective estimates, not ground truth.")
88
+ if report["usage"]["cost_usd"] is None:
89
+ notes.append("Cost is unavailable: no provider call reported a price.")
90
+ notes.append("Results come from one dataset split and depend on the models that were used.")
91
+ return notes
92
+
93
+
94
+ def _latest(
95
+ evals: EvaluationRepository, version: PromptVersion, partition: Partition
96
+ ) -> EvaluationRecord | None:
97
+ found = evals.find(prompt_number=version.number, partition=partition, limit=1)
98
+ return found[0] if found else None
99
+
100
+
101
+ def _dataset(datasets: DatasetRepository, dataset_id: int) -> dict[str, Any]:
102
+ info = datasets.dataset_info(dataset_id)
103
+ return {
104
+ "path": info.source_path,
105
+ "content_hash": info.content_hash,
106
+ "rows": info.row_count,
107
+ "skipped_rows": info.skipped_rows,
108
+ }
109
+
110
+
111
+ def _models(project: Project) -> dict[str, str | None]:
112
+ models = project.config.models
113
+ return {
114
+ role: (spec.model if spec else None)
115
+ for role, spec in (
116
+ ("target", models.target),
117
+ ("optimizer", models.optimizer),
118
+ ("analyzer", models.analyzer),
119
+ ("judge", models.judge),
120
+ )
121
+ }
122
+
123
+
124
+ def _metrics(project: Project) -> dict[str, Any]:
125
+ config = project.config.metrics
126
+ return {
127
+ "primary": config.primary,
128
+ "secondary": list(config.secondary),
129
+ "constraints": {k: b.model_dump(exclude_none=True) for k, b in config.constraints.items()},
130
+ "rules": sorted(config.rules),
131
+ "judges": sorted(config.judges),
132
+ "custom": config.python is not None,
133
+ }
134
+
135
+
136
+ def _per_class(record: EvaluationRecord | None) -> dict[str, Any]:
137
+ if record is None:
138
+ return {}
139
+ per_class: dict[str, Any] = record.report.primary_value.details.get("per_class", {})
140
+ return per_class
141
+
142
+
143
+ def _version(version: PromptVersion, decision: Any) -> dict[str, Any]:
144
+ return {
145
+ "version": version.number,
146
+ "parent": version.parent_number,
147
+ "source": version.source,
148
+ "change_summary": version.change_summary,
149
+ "status": decision.status if decision else None,
150
+ "primary_delta": decision.primary_delta if decision else None,
151
+ "decision_reason": decision.reason if decision else None,
152
+ }
153
+
154
+
155
+ def _run(run: RunRecord, runs: RunRepository) -> dict[str, Any]:
156
+ return {
157
+ "id": run.id,
158
+ "status": run.status,
159
+ "stop_reason": run.stop_reason,
160
+ "turns": len([t for t in runs.turns(run.id) if t.status == "done"]),
161
+ "model_calls": run.model_calls,
162
+ "cost_usd": run.cost_usd,
163
+ }
164
+
165
+
166
+ def _error_analysis(
167
+ project: Project, evals: EvaluationRepository, selected: PromptVersion
168
+ ) -> dict[str, Any] | None:
169
+ """Error counts from the latest validation outcomes; counts only, no example content."""
170
+ record = _latest(evals, selected, Partition.VALIDATION)
171
+ if record is None:
172
+ return None
173
+ outcomes = evals.outcomes(record.id)
174
+ facts = build_facts(outcomes, [], project.config.output.type, record.report.primary)
175
+ return {
176
+ "evaluation": record.id,
177
+ "examples": facts.total,
178
+ "failed": facts.failed,
179
+ "codes": dict(facts.codes),
180
+ "confusions": [{"expected": e, "got": g, "count": n} for e, g, n in facts.confusions],
181
+ }
182
+
183
+
184
+ def _usage(evals: EvaluationRepository) -> dict[str, Any]:
185
+ records = evals.find(limit=100000)
186
+ calls = sum(r.usage.calls - r.usage.cached_calls for r in records)
187
+ costs = [r.usage.cost_usd for r in records if r.usage.cost_usd is not None]
188
+ tokens = [r.usage.total_tokens for r in records if r.usage.total_tokens is not None]
189
+ return {
190
+ "evaluations": len(records),
191
+ "model_calls": calls,
192
+ "total_tokens": sum(tokens) if tokens else None,
193
+ "cost_usd": sum(costs) if costs else None,
194
+ }
@@ -0,0 +1,43 @@
1
+ """Human decisions on candidates that are waiting for approval."""
2
+
3
+ import sqlite3
4
+
5
+ from strop.errors import ConfigError
6
+ from strop.prompts.versions import parse_version_ref
7
+ from strop.storage.prompts import PromptRepository
8
+ from strop.storage.runs import CandidateRecord, RunRepository
9
+
10
+
11
+ def _pending(runs: RunRepository, ref: str) -> CandidateRecord:
12
+ number = parse_version_ref(ref)
13
+ candidate = runs.for_version(number)
14
+ if candidate is None or candidate.status != "pending":
15
+ raise ConfigError(f"v{number} is not waiting for a decision")
16
+ return candidate
17
+
18
+
19
+ def approve(conn: sqlite3.Connection, ref: str) -> CandidateRecord:
20
+ """Make a pending candidate the best prompt and close its turn."""
21
+ runs, prompts = RunRepository(conn), PromptRepository(conn)
22
+ candidate = _pending(runs, ref)
23
+ number = candidate.version_number
24
+ if number is None:
25
+ raise ConfigError(f"{ref} has no prompt version")
26
+ prompts.set_best(number)
27
+ runs.set_candidate_status(candidate.id, "accepted", f"{candidate.reason}; approved by the user")
28
+ runs.update_turn(candidate.run_id, candidate.turn, status="done", best_after=number)
29
+ runs.update_run(candidate.run_id, stale_turns=0)
30
+ return runs.get_candidate(candidate.id)
31
+
32
+
33
+ def reject(conn: sqlite3.Connection, ref: str, reason: str = "") -> CandidateRecord:
34
+ """Discard a pending candidate; the best prompt stays as it was."""
35
+ runs = RunRepository(conn)
36
+ candidate = _pending(runs, ref)
37
+ why = f"rejected by the user: {reason}" if reason else "rejected by the user"
38
+ runs.set_candidate_status(candidate.id, "rejected", why)
39
+ turn = runs.get_turn(candidate.run_id, candidate.turn)
40
+ runs.update_turn(candidate.run_id, candidate.turn, status="done", best_after=turn.best_before)
41
+ run = runs.get_run(candidate.run_id)
42
+ runs.update_run(candidate.run_id, stale_turns=run.stale_turns + 1)
43
+ return runs.get_candidate(candidate.id)
@@ -0,0 +1,79 @@
1
+ """Creating a new Strop project on disk."""
2
+
3
+ from datetime import UTC, datetime
4
+ from pathlib import Path
5
+
6
+ from strop import __version__
7
+ from strop.config import OutputType, StropConfig, dump_config
8
+ from strop.errors import ConfigError
9
+ from strop.evaluation.parsing import load_output_schema
10
+ from strop.prompts import PromptMessage, PromptTemplate, dump_prompt
11
+ from strop.services.metrics import validate_metric_names
12
+ from strop.storage import ProjectPaths, open_database
13
+
14
+
15
+ def scaffold_project(config: StropConfig, root: Path, *, force: bool = False) -> list[str]:
16
+ """Write strop.yaml, a starter prompt if none exists, and the state directory.
17
+
18
+ Returns the entries that were created or rewritten, relative to ``root``.
19
+ """
20
+ dataset_file = root / config.dataset.path
21
+ if not dataset_file.is_file():
22
+ raise ConfigError(f"dataset file not found: {dataset_file}")
23
+ load_output_schema(root, config.output)
24
+ validate_metric_names(config)
25
+ if config.metrics.python is not None and not (root / config.metrics.python.file).is_file():
26
+ raise ConfigError(f"metric file not found: {root / config.metrics.python.file}")
27
+
28
+ paths = ProjectPaths(root)
29
+ dump_config(config, paths.config_file, force=force)
30
+ created = [paths.config_file.name]
31
+
32
+ prompt_file = root / config.prompt.file
33
+ if not prompt_file.exists():
34
+ prompt_file.parent.mkdir(parents=True, exist_ok=True)
35
+ prompt_file.write_text(starter_prompt(config), encoding="utf-8")
36
+ created.append(str(config.prompt.file))
37
+
38
+ conn = open_database(paths)
39
+ try:
40
+ conn.executemany(
41
+ "INSERT OR REPLACE INTO meta (key, value) VALUES (?, ?)",
42
+ [
43
+ ("project_name", config.project.name),
44
+ ("strop_version", __version__),
45
+ ("created_at", datetime.now(UTC).isoformat()),
46
+ ],
47
+ )
48
+ finally:
49
+ conn.close()
50
+ created.append(f"{paths.state_dir.name}/")
51
+ return created
52
+
53
+
54
+ def data_recipients(config: StropConfig) -> list[str]:
55
+ """Provider names that will receive dataset content, from the configured models."""
56
+ models = config.models
57
+ specs = (models.target, models.optimizer, models.analyzer, models.judge)
58
+ return sorted({spec.model.split("/", 1)[0] for spec in specs if spec is not None})
59
+
60
+
61
+ def starter_prompt(config: StropConfig) -> str:
62
+ """A minimal prompt file the user is expected to edit before running.
63
+
64
+ The format follows the prompt file extension: YAML messages for ``.yaml``/``.yml``,
65
+ a single plain-text template otherwise.
66
+ """
67
+ system = config.project.task_description.strip()
68
+ if config.output.type is OutputType.LABEL:
69
+ labels = ", ".join(config.output.labels)
70
+ system += f"\nRespond with exactly one of these labels and nothing else: {labels}."
71
+ elif config.output.type is OutputType.JSON:
72
+ system += "\nRespond with a single valid JSON object and nothing else."
73
+ user = "\n".join(f"{name}: {{{{{name}}}}}" for name in config.dataset.input_columns)
74
+ if config.prompt.file.suffix.lower() not in {".yaml", ".yml"}:
75
+ return f"{system}\n\n{user}\n"
76
+ template = PromptTemplate(
77
+ (PromptMessage("system", system), PromptMessage("user", user)), frozenset({"system"})
78
+ )
79
+ return dump_prompt(template)