strop-cli 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- strop/__init__.py +5 -0
- strop/__main__.py +5 -0
- strop/analysis/__init__.py +17 -0
- strop/analysis/classify.py +52 -0
- strop/analysis/evidence.py +52 -0
- strop/analysis/facts.py +74 -0
- strop/analysis/patterns.py +90 -0
- strop/cli/__init__.py +5 -0
- strop/cli/analyze.py +93 -0
- strop/cli/app.py +82 -0
- strop/cli/data.py +111 -0
- strop/cli/evaluate.py +145 -0
- strop/cli/export.py +74 -0
- strop/cli/final.py +106 -0
- strop/cli/init.py +131 -0
- strop/cli/optimize.py +156 -0
- strop/cli/options.py +20 -0
- strop/cli/progress.py +18 -0
- strop/cli/prompts.py +110 -0
- strop/cli/status.py +68 -0
- strop/cli/versions.py +123 -0
- strop/config/__init__.py +16 -0
- strop/config/base.py +9 -0
- strop/config/constraints.py +33 -0
- strop/config/dataset.py +63 -0
- strop/config/enums.py +52 -0
- strop/config/io.py +52 -0
- strop/config/limits.py +19 -0
- strop/config/llm.py +42 -0
- strop/config/metrics.py +86 -0
- strop/config/optimization.py +23 -0
- strop/config/output.py +31 -0
- strop/config/overrides.py +22 -0
- strop/config/root.py +36 -0
- strop/config/rules.py +67 -0
- strop/config/split.py +23 -0
- strop/data/__init__.py +58 -0
- strop/data/loaders.py +85 -0
- strop/data/models.py +44 -0
- strop/data/rows.py +99 -0
- strop/data/splitting.py +103 -0
- strop/env.py +44 -0
- strop/errors.py +59 -0
- strop/evaluation/__init__.py +1 -0
- strop/evaluation/budget.py +67 -0
- strop/evaluation/budgeted.py +23 -0
- strop/evaluation/examples.py +49 -0
- strop/evaluation/models.py +105 -0
- strop/evaluation/parsing.py +96 -0
- strop/evaluation/runner.py +107 -0
- strop/formatting.py +14 -0
- strop/llm/__init__.py +7 -0
- strop/llm/caching.py +28 -0
- strop/llm/client.py +15 -0
- strop/llm/factory.py +14 -0
- strop/llm/litellm_client.py +90 -0
- strop/llm/mock.py +43 -0
- strop/llm/retry.py +44 -0
- strop/llm/structured.py +52 -0
- strop/llm/types.py +59 -0
- strop/log.py +49 -0
- strop/metrics/__init__.py +1 -0
- strop/metrics/basic.py +30 -0
- strop/metrics/classification.py +164 -0
- strop/metrics/judge.py +130 -0
- strop/metrics/python_metric.py +101 -0
- strop/metrics/python_runner.py +65 -0
- strop/metrics/registry.py +115 -0
- strop/metrics/rules.py +55 -0
- strop/metrics/structured.py +109 -0
- strop/metrics/suite.py +109 -0
- strop/metrics/text.py +55 -0
- strop/metrics/types.py +106 -0
- strop/optimization/__init__.py +25 -0
- strop/optimization/acceptance.py +170 -0
- strop/optimization/generator.py +92 -0
- strop/optimization/meta_prompt.py +101 -0
- strop/optimization/types.py +69 -0
- strop/prompts/__init__.py +18 -0
- strop/prompts/diff.py +139 -0
- strop/prompts/files.py +92 -0
- strop/prompts/models.py +51 -0
- strop/prompts/render.py +16 -0
- strop/prompts/template.py +45 -0
- strop/prompts/validation.py +31 -0
- strop/prompts/versions.py +34 -0
- strop/py.typed +0 -0
- strop/reporting/__init__.py +6 -0
- strop/reporting/console.py +20 -0
- strop/reporting/diff_view.py +102 -0
- strop/reporting/evaluation_view.py +141 -0
- strop/reporting/final_view.py +109 -0
- strop/reporting/prompt_export.py +58 -0
- strop/reporting/report_markdown.py +157 -0
- strop/reporting/run_view.py +118 -0
- strop/services/__init__.py +1 -0
- strop/services/analysis.py +124 -0
- strop/services/clean.py +38 -0
- strop/services/compare.py +60 -0
- strop/services/context.py +70 -0
- strop/services/data.py +65 -0
- strop/services/evaluate.py +105 -0
- strop/services/evaluator.py +169 -0
- strop/services/export.py +90 -0
- strop/services/final_test.py +69 -0
- strop/services/metrics.py +89 -0
- strop/services/optimize.py +187 -0
- strop/services/partitions.py +46 -0
- strop/services/project.py +55 -0
- strop/services/prompts.py +76 -0
- strop/services/report.py +194 -0
- strop/services/review.py +43 -0
- strop/services/scaffold.py +79 -0
- strop/services/turn.py +234 -0
- strop/storage/__init__.py +6 -0
- strop/storage/cache.py +45 -0
- strop/storage/database.py +45 -0
- strop/storage/datasets.py +129 -0
- strop/storage/evaluations.py +160 -0
- strop/storage/migrations/001_initial.sql +5 -0
- strop/storage/migrations/002_datasets.sql +27 -0
- strop/storage/migrations/003_prompt_versions.sql +22 -0
- strop/storage/migrations/004_llm_cache.sql +6 -0
- strop/storage/migrations/005_evaluations.sql +37 -0
- strop/storage/migrations/006_runs.sql +52 -0
- strop/storage/migrations/007_example_feedback.sql +2 -0
- strop/storage/migrations/__init__.py +5 -0
- strop/storage/migrator.py +91 -0
- strop/storage/paths.py +30 -0
- strop/storage/prompts.py +96 -0
- strop/storage/run_models.py +75 -0
- strop/storage/runs.py +263 -0
- strop_cli-0.1.0.dist-info/METADATA +242 -0
- strop_cli-0.1.0.dist-info/RECORD +137 -0
- strop_cli-0.1.0.dist-info/WHEEL +4 -0
- strop_cli-0.1.0.dist-info/entry_points.txt +2 -0
- strop_cli-0.1.0.dist-info/licenses/LICENSE +21 -0
strop/cli/evaluate.py
ADDED
|
@@ -0,0 +1,145 @@
|
|
|
1
|
+
"""The ``baseline`` and ``eval`` commands."""
|
|
2
|
+
|
|
3
|
+
import asyncio
|
|
4
|
+
from enum import StrEnum
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
from typing import Annotated
|
|
7
|
+
|
|
8
|
+
import typer
|
|
9
|
+
|
|
10
|
+
from strop.cli.options import DirOption, JsonOption, emit_json
|
|
11
|
+
from strop.cli.progress import stderr_progress
|
|
12
|
+
from strop.data.models import Partition
|
|
13
|
+
from strop.evaluation.models import EvaluationRecord
|
|
14
|
+
from strop.formatting import fmt
|
|
15
|
+
from strop.reporting import ColorMode, make_console
|
|
16
|
+
from strop.reporting.evaluation_view import record_payload, render_evaluation
|
|
17
|
+
from strop.services.evaluate import BaselineOutcome, run_baseline, run_eval
|
|
18
|
+
from strop.services.project import Project, open_project
|
|
19
|
+
from strop.services.prompts import has_unregistered_edits
|
|
20
|
+
from strop.storage.evaluations import EvaluationRepository
|
|
21
|
+
from strop.storage.prompts import PromptRepository
|
|
22
|
+
|
|
23
|
+
NoCacheOption = Annotated[bool, typer.Option("--no-cache", help="Do not read or write the cache.")]
|
|
24
|
+
SampleOption = Annotated[
|
|
25
|
+
int | None, typer.Option(help="Use a seeded random sample of this many rows.", min=1)
|
|
26
|
+
]
|
|
27
|
+
ShowErrorsOption = Annotated[int, typer.Option(help="How many failing examples to list.", min=0)]
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
class EvalPartition(StrEnum):
|
|
31
|
+
"""Partitions that may be evaluated freely. The test partition is not among them."""
|
|
32
|
+
|
|
33
|
+
TRAIN = "train"
|
|
34
|
+
VALIDATION = "validation"
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def baseline_command(
|
|
38
|
+
repeats: Annotated[int, typer.Option(help="Run the baseline this many times.", min=1)] = 1,
|
|
39
|
+
sample: SampleOption = None,
|
|
40
|
+
with_train: Annotated[bool, typer.Option(help="Also evaluate on the train partition.")] = False,
|
|
41
|
+
no_cache: NoCacheOption = False,
|
|
42
|
+
show_errors: ShowErrorsOption = 5,
|
|
43
|
+
as_json: JsonOption = False,
|
|
44
|
+
directory: DirOption = Path(),
|
|
45
|
+
) -> None:
|
|
46
|
+
"""Measure the initial prompt (v1) on the validation partition."""
|
|
47
|
+
project = open_project(directory)
|
|
48
|
+
with project.database() as conn:
|
|
49
|
+
progress = None if as_json else stderr_progress("Baseline")
|
|
50
|
+
outcome = asyncio.run(
|
|
51
|
+
run_baseline(
|
|
52
|
+
project,
|
|
53
|
+
conn,
|
|
54
|
+
repeats=repeats,
|
|
55
|
+
sample=sample,
|
|
56
|
+
with_train=with_train,
|
|
57
|
+
use_cache=not no_cache,
|
|
58
|
+
progress=progress,
|
|
59
|
+
)
|
|
60
|
+
)
|
|
61
|
+
repo = EvaluationRepository(conn)
|
|
62
|
+
_warn_about_edits(project, PromptRepository(conn))
|
|
63
|
+
if as_json:
|
|
64
|
+
emit_json(_baseline_payload(outcome))
|
|
65
|
+
return
|
|
66
|
+
console = make_console(ColorMode.AUTO)
|
|
67
|
+
for record in [outcome.reference, *([outcome.train] if outcome.train else [])]:
|
|
68
|
+
render_evaluation(
|
|
69
|
+
console,
|
|
70
|
+
record,
|
|
71
|
+
repo.outcomes(record.id),
|
|
72
|
+
version_ref=outcome.version.ref,
|
|
73
|
+
show_errors=show_errors,
|
|
74
|
+
)
|
|
75
|
+
if outcome.noise is not None:
|
|
76
|
+
typer.echo(
|
|
77
|
+
f"Run-to-run noise over {repeats} runs: the primary metric varied by "
|
|
78
|
+
f"{fmt(outcome.noise)}. Improvements smaller than this are not meaningful."
|
|
79
|
+
)
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def eval_command(
|
|
83
|
+
version: Annotated[
|
|
84
|
+
str | None, typer.Argument(help="Version such as v2; default: best.")
|
|
85
|
+
] = None,
|
|
86
|
+
partition: Annotated[EvalPartition, typer.Option(help="Partition to evaluate.")] = (
|
|
87
|
+
EvalPartition.VALIDATION
|
|
88
|
+
),
|
|
89
|
+
sample: SampleOption = None,
|
|
90
|
+
no_cache: NoCacheOption = False,
|
|
91
|
+
show_errors: ShowErrorsOption = 5,
|
|
92
|
+
as_json: JsonOption = False,
|
|
93
|
+
directory: DirOption = Path(),
|
|
94
|
+
) -> None:
|
|
95
|
+
"""Score a prompt version on a partition. Nothing is changed or promoted."""
|
|
96
|
+
project = open_project(directory)
|
|
97
|
+
with project.database() as conn:
|
|
98
|
+
selected, record = asyncio.run(
|
|
99
|
+
run_eval(
|
|
100
|
+
project,
|
|
101
|
+
conn,
|
|
102
|
+
ref=version,
|
|
103
|
+
partition=Partition(partition.value),
|
|
104
|
+
sample=sample,
|
|
105
|
+
use_cache=not no_cache,
|
|
106
|
+
progress=None if as_json else stderr_progress("Evaluating"),
|
|
107
|
+
)
|
|
108
|
+
)
|
|
109
|
+
outcomes = EvaluationRepository(conn).outcomes(record.id)
|
|
110
|
+
if version is None:
|
|
111
|
+
_warn_about_edits(project, PromptRepository(conn))
|
|
112
|
+
if as_json:
|
|
113
|
+
emit_json(record_payload(record, selected.ref))
|
|
114
|
+
return
|
|
115
|
+
render_evaluation(
|
|
116
|
+
make_console(ColorMode.AUTO),
|
|
117
|
+
record,
|
|
118
|
+
outcomes,
|
|
119
|
+
version_ref=selected.ref,
|
|
120
|
+
show_errors=show_errors,
|
|
121
|
+
)
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
def _warn_about_edits(project: Project, repo: PromptRepository) -> None:
|
|
125
|
+
if has_unregistered_edits(project, repo):
|
|
126
|
+
typer.echo(
|
|
127
|
+
f"Warning: {project.config.prompt.file} differs from every stored prompt version. "
|
|
128
|
+
f"Run 'strop prompt add --file {project.config.prompt.file} --use' to use your edits.",
|
|
129
|
+
err=True,
|
|
130
|
+
)
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
def _baseline_payload(outcome: BaselineOutcome) -> dict[str, object]:
|
|
134
|
+
ref = outcome.version.ref
|
|
135
|
+
|
|
136
|
+
def one(record: EvaluationRecord) -> dict[str, object]:
|
|
137
|
+
return dict(record_payload(record, ref))
|
|
138
|
+
|
|
139
|
+
return {
|
|
140
|
+
"version": ref,
|
|
141
|
+
"baseline": one(outcome.reference),
|
|
142
|
+
"repeats": [one(r) for r in outcome.records],
|
|
143
|
+
"noise": outcome.noise,
|
|
144
|
+
"train": one(outcome.train) if outcome.train else None,
|
|
145
|
+
}
|
strop/cli/export.py
ADDED
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
"""The ``export`` and ``clean`` commands."""
|
|
2
|
+
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
from typing import Annotated
|
|
5
|
+
|
|
6
|
+
import typer
|
|
7
|
+
|
|
8
|
+
from strop.cli.options import DirOption, JsonOption, emit_json
|
|
9
|
+
from strop.reporting.prompt_export import PromptFormat
|
|
10
|
+
from strop.services.clean import clean_project
|
|
11
|
+
from strop.services.export import export_project
|
|
12
|
+
from strop.services.project import open_project
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def export_command(
|
|
16
|
+
version: Annotated[
|
|
17
|
+
str | None, typer.Argument(help="Version such as v2; default: best.")
|
|
18
|
+
] = None,
|
|
19
|
+
out: Annotated[Path, typer.Option(help="Output directory, relative to the project.")] = Path(
|
|
20
|
+
"exports"
|
|
21
|
+
),
|
|
22
|
+
prompt_format: Annotated[
|
|
23
|
+
PromptFormat, typer.Option(help="Format of the exported prompt file.")
|
|
24
|
+
] = PromptFormat.MD,
|
|
25
|
+
results: Annotated[
|
|
26
|
+
bool, typer.Option(help="Also write results.csv with per-example model outputs.")
|
|
27
|
+
] = True,
|
|
28
|
+
write_prompt: Annotated[
|
|
29
|
+
Path | None, typer.Option(help="Also write the prompt as a Strop prompt YAML file here.")
|
|
30
|
+
] = None,
|
|
31
|
+
force: Annotated[bool, typer.Option(help="Overwrite existing files.")] = False,
|
|
32
|
+
as_json: JsonOption = False,
|
|
33
|
+
directory: DirOption = Path(),
|
|
34
|
+
) -> None:
|
|
35
|
+
"""Export a prompt version, a JSON and Markdown report, and per-example results."""
|
|
36
|
+
project = open_project(directory)
|
|
37
|
+
with project.database() as conn:
|
|
38
|
+
result = export_project(
|
|
39
|
+
project,
|
|
40
|
+
conn,
|
|
41
|
+
ref=version,
|
|
42
|
+
out=directory / out,
|
|
43
|
+
prompt_format=prompt_format,
|
|
44
|
+
include_results=results,
|
|
45
|
+
write_prompt=directory / write_prompt if write_prompt else None,
|
|
46
|
+
force=force,
|
|
47
|
+
)
|
|
48
|
+
if as_json:
|
|
49
|
+
emit_json({"version": result.version.number, "files": [str(p) for p in result.files]})
|
|
50
|
+
return
|
|
51
|
+
typer.echo(f"Exported {result.version.ref}:")
|
|
52
|
+
for path in result.files:
|
|
53
|
+
typer.echo(f" {path}")
|
|
54
|
+
if results:
|
|
55
|
+
typer.echo("Note: results.csv contains model outputs for your dataset rows.")
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def clean_command(
|
|
59
|
+
cache: Annotated[bool, typer.Option("--cache", help="Delete cached model responses.")] = False,
|
|
60
|
+
outputs: Annotated[
|
|
61
|
+
bool, typer.Option("--outputs", help="Delete stored model outputs but keep the scores.")
|
|
62
|
+
] = False,
|
|
63
|
+
everything: Annotated[
|
|
64
|
+
bool, typer.Option("--all", help="Delete the whole .strop directory.")
|
|
65
|
+
] = False,
|
|
66
|
+
yes: Annotated[bool, typer.Option("--yes", help="Confirm the deletion.")] = False,
|
|
67
|
+
directory: DirOption = Path(),
|
|
68
|
+
) -> None:
|
|
69
|
+
"""Delete stored data. strop.yaml, the prompt file, and the dataset are never touched."""
|
|
70
|
+
project = open_project(directory)
|
|
71
|
+
for line in clean_project(
|
|
72
|
+
project, cache=cache, outputs=outputs, everything=everything, confirmed=yes
|
|
73
|
+
):
|
|
74
|
+
typer.echo(line)
|
strop/cli/final.py
ADDED
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
"""The ``test`` and ``compare`` commands."""
|
|
2
|
+
|
|
3
|
+
import asyncio
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
from typing import Annotated
|
|
6
|
+
|
|
7
|
+
import typer
|
|
8
|
+
|
|
9
|
+
from strop.cli.evaluate import EvalPartition, NoCacheOption, SampleOption, ShowErrorsOption
|
|
10
|
+
from strop.cli.options import DirOption, JsonOption, emit_json
|
|
11
|
+
from strop.cli.progress import stderr_progress
|
|
12
|
+
from strop.data.models import Partition
|
|
13
|
+
from strop.reporting import ColorMode, make_console
|
|
14
|
+
from strop.reporting.evaluation_view import render_evaluation
|
|
15
|
+
from strop.reporting.final_view import (
|
|
16
|
+
comparison_payload,
|
|
17
|
+
final_test_payload,
|
|
18
|
+
render_comparison,
|
|
19
|
+
render_final_test,
|
|
20
|
+
)
|
|
21
|
+
from strop.services.compare import parse_models, run_compare
|
|
22
|
+
from strop.services.final_test import run_final_test
|
|
23
|
+
from strop.services.project import open_project
|
|
24
|
+
from strop.storage.evaluations import EvaluationRepository
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def final_test_command(
|
|
28
|
+
version: Annotated[
|
|
29
|
+
str | None, typer.Argument(help="Version such as v2; default: best.")
|
|
30
|
+
] = None,
|
|
31
|
+
confirm: Annotated[
|
|
32
|
+
bool, typer.Option("--confirm", help="Acknowledge that this uses up the test set.")
|
|
33
|
+
] = False,
|
|
34
|
+
allow_rerun: Annotated[
|
|
35
|
+
bool, typer.Option("--allow-rerun", help="Allow a second test evaluation.")
|
|
36
|
+
] = False,
|
|
37
|
+
with_baseline: Annotated[
|
|
38
|
+
bool, typer.Option("--with-baseline", help="Also test the initial prompt (v1).")
|
|
39
|
+
] = False,
|
|
40
|
+
no_cache: NoCacheOption = False,
|
|
41
|
+
show_errors: ShowErrorsOption = 5,
|
|
42
|
+
as_json: JsonOption = False,
|
|
43
|
+
directory: DirOption = Path(),
|
|
44
|
+
) -> None:
|
|
45
|
+
"""Run the final evaluation on the held-out test partition.
|
|
46
|
+
|
|
47
|
+
The test set is only a fair estimate if it is looked at once. Pass --confirm to proceed.
|
|
48
|
+
"""
|
|
49
|
+
project = open_project(directory)
|
|
50
|
+
with project.database() as conn:
|
|
51
|
+
result = asyncio.run(
|
|
52
|
+
run_final_test(
|
|
53
|
+
project,
|
|
54
|
+
conn,
|
|
55
|
+
ref=version,
|
|
56
|
+
confirm=confirm,
|
|
57
|
+
allow_rerun=allow_rerun,
|
|
58
|
+
with_baseline=with_baseline,
|
|
59
|
+
use_cache=not no_cache,
|
|
60
|
+
progress=None if as_json else stderr_progress("Testing"),
|
|
61
|
+
)
|
|
62
|
+
)
|
|
63
|
+
outcomes = EvaluationRepository(conn).outcomes(result.record.id)
|
|
64
|
+
if as_json:
|
|
65
|
+
emit_json(final_test_payload(result))
|
|
66
|
+
return
|
|
67
|
+
console = make_console(ColorMode.AUTO)
|
|
68
|
+
render_evaluation(
|
|
69
|
+
console, result.record, outcomes, version_ref=result.version.ref, show_errors=show_errors
|
|
70
|
+
)
|
|
71
|
+
render_final_test(console, result)
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def compare_command(
|
|
75
|
+
models: Annotated[str, typer.Option(help="Comma-separated target models to compare.")],
|
|
76
|
+
version: Annotated[
|
|
77
|
+
str | None, typer.Argument(help="Version such as v2; default: best.")
|
|
78
|
+
] = None,
|
|
79
|
+
partition: Annotated[EvalPartition, typer.Option(help="Partition to evaluate.")] = (
|
|
80
|
+
EvalPartition.VALIDATION
|
|
81
|
+
),
|
|
82
|
+
sample: SampleOption = None,
|
|
83
|
+
no_cache: NoCacheOption = False,
|
|
84
|
+
as_json: JsonOption = False,
|
|
85
|
+
directory: DirOption = Path(),
|
|
86
|
+
) -> None:
|
|
87
|
+
"""Run one prompt version on several models and compare the results."""
|
|
88
|
+
names = parse_models(models)
|
|
89
|
+
project = open_project(directory)
|
|
90
|
+
with project.database() as conn:
|
|
91
|
+
comparison = asyncio.run(
|
|
92
|
+
run_compare(
|
|
93
|
+
project,
|
|
94
|
+
conn,
|
|
95
|
+
models=names,
|
|
96
|
+
ref=version,
|
|
97
|
+
partition=Partition(partition.value),
|
|
98
|
+
sample=sample,
|
|
99
|
+
use_cache=not no_cache,
|
|
100
|
+
progress=None if as_json else stderr_progress("Comparing"),
|
|
101
|
+
)
|
|
102
|
+
)
|
|
103
|
+
if as_json:
|
|
104
|
+
emit_json(comparison_payload(comparison))
|
|
105
|
+
return
|
|
106
|
+
render_comparison(make_console(ColorMode.AUTO), comparison)
|
strop/cli/init.py
ADDED
|
@@ -0,0 +1,131 @@
|
|
|
1
|
+
"""The ``strop init`` command."""
|
|
2
|
+
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
from typing import Annotated, Any
|
|
5
|
+
|
|
6
|
+
import typer
|
|
7
|
+
from pydantic import ValidationError
|
|
8
|
+
|
|
9
|
+
from strop.cli.options import DirOption
|
|
10
|
+
from strop.config import DatasetFormat, OutputType, StropConfig
|
|
11
|
+
from strop.config.constraints import parse_constraints
|
|
12
|
+
from strop.config.io import format_errors
|
|
13
|
+
from strop.config.rules import load_rubrics, parse_rules
|
|
14
|
+
from strop.errors import ConfigError
|
|
15
|
+
from strop.services.scaffold import data_recipients, scaffold_project
|
|
16
|
+
|
|
17
|
+
Option = typer.Option
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def init_command(
|
|
21
|
+
name: Annotated[str, Option(help="Project name.")],
|
|
22
|
+
task: Annotated[str, Option(help="One-sentence description of what the prompt must do.")],
|
|
23
|
+
data: Annotated[Path, Option(help="Dataset file (CSV or JSONL), relative to the project.")],
|
|
24
|
+
input_columns: Annotated[
|
|
25
|
+
list[str], Option("--input", help="Dataset column used as a prompt variable. Repeatable.")
|
|
26
|
+
],
|
|
27
|
+
output_type: Annotated[OutputType, Option(help="Kind of output the model must produce.")],
|
|
28
|
+
metric: Annotated[str, Option(help="Primary metric, for example macro_f1.")],
|
|
29
|
+
target_model: Annotated[str, Option(help="Model the prompt is optimized for (LiteLLM name).")],
|
|
30
|
+
expected: Annotated[
|
|
31
|
+
str | None, Option(help="Dataset column holding the expected output.")
|
|
32
|
+
] = None,
|
|
33
|
+
metadata: Annotated[
|
|
34
|
+
list[str] | None, Option(help="Dataset column kept as metadata. Repeatable.")
|
|
35
|
+
] = None,
|
|
36
|
+
labels: Annotated[str, Option(help="Comma-separated labels, for output type 'label'.")] = "",
|
|
37
|
+
output_schema: Annotated[
|
|
38
|
+
Path | None, Option(help="JSON schema file the output must satisfy, for type 'json'.")
|
|
39
|
+
] = None,
|
|
40
|
+
secondary_metric: Annotated[
|
|
41
|
+
list[str] | None, Option(help="Additional metric to report. Repeatable.")
|
|
42
|
+
] = None,
|
|
43
|
+
constraint: Annotated[
|
|
44
|
+
list[str] | None, Option(help="Metric limit such as 'valid_output_rate>=0.98'. Repeatable.")
|
|
45
|
+
] = None,
|
|
46
|
+
rule: Annotated[
|
|
47
|
+
list[str] | None, Option(help="Output check such as 'short=max_words:120'. Repeatable.")
|
|
48
|
+
] = None,
|
|
49
|
+
metric_file: Annotated[
|
|
50
|
+
Path | None, Option(help="Python file defining score(output, row); metric name 'custom'.")
|
|
51
|
+
] = None,
|
|
52
|
+
rubric_file: Annotated[
|
|
53
|
+
Path | None, Option(help="YAML file of judge rubrics; each becomes metric 'judge:<name>'.")
|
|
54
|
+
] = None,
|
|
55
|
+
optimizer_model: Annotated[str | None, Option(help="Model that proposes revisions.")] = None,
|
|
56
|
+
analyzer_model: Annotated[str | None, Option(help="Model that summarizes failures.")] = None,
|
|
57
|
+
judge_model: Annotated[str | None, Option(help="Model that scores rubric metrics.")] = None,
|
|
58
|
+
prompt: Annotated[Path, Option(help="Prompt file to create or use.")] = Path("prompt.yaml"),
|
|
59
|
+
data_format: Annotated[
|
|
60
|
+
DatasetFormat | None, Option("--format", help="Dataset format; inferred if omitted.")
|
|
61
|
+
] = None,
|
|
62
|
+
id_column: Annotated[str | None, Option(help="Dataset column with a unique row id.")] = None,
|
|
63
|
+
seed: Annotated[int, Option(help="Random seed for data splits.")] = 42,
|
|
64
|
+
directory: DirOption = Path(),
|
|
65
|
+
force: Annotated[bool, Option(help="Overwrite an existing strop.yaml.")] = False,
|
|
66
|
+
) -> None:
|
|
67
|
+
"""Create strop.yaml, a starter prompt, and the local .strop/ state directory."""
|
|
68
|
+
raw = {
|
|
69
|
+
"project": {"name": name, "task_description": task},
|
|
70
|
+
"prompt": {"file": str(prompt)},
|
|
71
|
+
"dataset": {
|
|
72
|
+
"path": str(data),
|
|
73
|
+
"format": data_format,
|
|
74
|
+
"id_column": id_column,
|
|
75
|
+
"columns": _column_roles(input_columns, expected, metadata or []),
|
|
76
|
+
},
|
|
77
|
+
"output": {
|
|
78
|
+
"type": output_type,
|
|
79
|
+
"labels": [item.strip() for item in labels.split(",") if item.strip()],
|
|
80
|
+
"schema_file": str(output_schema) if output_schema else None,
|
|
81
|
+
},
|
|
82
|
+
"models": _models(target_model, optimizer_model, analyzer_model, judge_model),
|
|
83
|
+
"metrics": {
|
|
84
|
+
"primary": metric,
|
|
85
|
+
"secondary": secondary_metric or [],
|
|
86
|
+
"constraints": {
|
|
87
|
+
k: v.model_dump() for k, v in parse_constraints(constraint or []).items()
|
|
88
|
+
},
|
|
89
|
+
"rules": {k: v.model_dump(mode="json") for k, v in parse_rules(rule or []).items()},
|
|
90
|
+
"python": {"file": str(metric_file)} if metric_file else None,
|
|
91
|
+
"judges": _rubrics(directory, rubric_file),
|
|
92
|
+
},
|
|
93
|
+
"split": {"seed": seed},
|
|
94
|
+
}
|
|
95
|
+
config = _validate(raw)
|
|
96
|
+
for entry in scaffold_project(config, directory, force=force):
|
|
97
|
+
typer.echo(f"Created {entry}")
|
|
98
|
+
typer.echo(f"Note: dataset rows will be sent to: {', '.join(data_recipients(config))}")
|
|
99
|
+
typer.echo("Note: keep .strop/ and .env out of version control; they hold outputs and keys.")
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
def _column_roles(inputs: list[str], expected: str | None, metadata: list[str]) -> dict[str, str]:
|
|
103
|
+
roles: dict[str, str] = {}
|
|
104
|
+
pairs = [(c, "input") for c in inputs] + [(c, "metadata") for c in metadata]
|
|
105
|
+
if expected:
|
|
106
|
+
pairs.append((expected, "expected"))
|
|
107
|
+
for column, role in pairs:
|
|
108
|
+
if column in roles:
|
|
109
|
+
raise ConfigError(f"column '{column}' is given more than one role")
|
|
110
|
+
roles[column] = role
|
|
111
|
+
return roles
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
def _models(
|
|
115
|
+
target: str, optimizer: str | None, analyzer: str | None, judge: str | None
|
|
116
|
+
) -> dict[str, Any]:
|
|
117
|
+
chosen = {"target": target, "optimizer": optimizer, "analyzer": analyzer, "judge": judge}
|
|
118
|
+
return {role: {"model": model} for role, model in chosen.items() if model}
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
def _validate(raw: dict[str, Any]) -> StropConfig:
|
|
122
|
+
try:
|
|
123
|
+
return StropConfig.model_validate(raw)
|
|
124
|
+
except ValidationError as exc:
|
|
125
|
+
raise ConfigError(f"invalid options:\n{format_errors(exc)}") from exc
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
def _rubrics(directory: Path, file: Path | None) -> dict[str, Any]:
|
|
129
|
+
if file is None:
|
|
130
|
+
return {}
|
|
131
|
+
return {name: spec.model_dump() for name, spec in load_rubrics(directory / file).items()}
|
strop/cli/optimize.py
ADDED
|
@@ -0,0 +1,156 @@
|
|
|
1
|
+
"""The ``optimize``, ``approve`` and ``reject`` commands."""
|
|
2
|
+
|
|
3
|
+
import asyncio
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
from typing import Annotated
|
|
6
|
+
|
|
7
|
+
import typer
|
|
8
|
+
|
|
9
|
+
from strop.cli.evaluate import NoCacheOption
|
|
10
|
+
from strop.cli.options import DirOption, JsonOption, emit_json
|
|
11
|
+
from strop.config.enums import ApprovalMode
|
|
12
|
+
from strop.config.overrides import apply_overrides
|
|
13
|
+
from strop.errors import ConfigError, ExitCode
|
|
14
|
+
from strop.reporting import ColorMode, make_console
|
|
15
|
+
from strop.reporting.run_view import render_run, run_payload
|
|
16
|
+
from strop.services.optimize import RunSummary, optimize
|
|
17
|
+
from strop.services.project import Project, open_project
|
|
18
|
+
from strop.services.review import approve, reject
|
|
19
|
+
|
|
20
|
+
Option = typer.Option
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def optimize_command(
|
|
24
|
+
max_turns: Annotated[int | None, Option(help="Maximum number of turns.", min=1)] = None,
|
|
25
|
+
candidates: Annotated[int | None, Option(help="Candidates per turn.", min=1)] = None,
|
|
26
|
+
target: Annotated[
|
|
27
|
+
float | None, Option(help="Stop once the primary metric reaches this.")
|
|
28
|
+
] = None,
|
|
29
|
+
patience: Annotated[
|
|
30
|
+
int | None, Option(help="Stop after this many turns without gain.", min=1)
|
|
31
|
+
] = None,
|
|
32
|
+
approval: Annotated[
|
|
33
|
+
ApprovalMode | None, Option(help="auto, or manual to pause for approval.")
|
|
34
|
+
] = None,
|
|
35
|
+
instructions: Annotated[str | None, Option(help="Extra guidance for the optimizer.")] = None,
|
|
36
|
+
feedback_file: Annotated[Path | None, Option(help="Text file with extra guidance.")] = None,
|
|
37
|
+
train_sample: Annotated[
|
|
38
|
+
int | None, Option(help="Analyze a sample of train rows.", min=1)
|
|
39
|
+
] = None,
|
|
40
|
+
max_calls: Annotated[int | None, Option(help="Model call budget for this run.", min=1)] = None,
|
|
41
|
+
max_cost: Annotated[float | None, Option(help="Cost budget in USD.", min=0.0)] = None,
|
|
42
|
+
max_minutes: Annotated[float | None, Option(help="Time budget in minutes.", min=0.0)] = None,
|
|
43
|
+
optimizer_model: Annotated[str | None, Option(help="Use this model as the optimizer.")] = None,
|
|
44
|
+
analyzer_model: Annotated[str | None, Option(help="Use this model as the analyzer.")] = None,
|
|
45
|
+
resume: Annotated[bool, Option(help="Continue the unfinished run.")] = False,
|
|
46
|
+
no_cache: NoCacheOption = False,
|
|
47
|
+
as_json: JsonOption = False,
|
|
48
|
+
directory: DirOption = Path(),
|
|
49
|
+
) -> None:
|
|
50
|
+
"""Run the optimization loop: test the prompt, revise it from the failures, and repeat."""
|
|
51
|
+
project = open_project(directory).with_models(
|
|
52
|
+
{"optimizer": optimizer_model, "analyzer": analyzer_model}
|
|
53
|
+
)
|
|
54
|
+
settings = apply_overrides(
|
|
55
|
+
project.config.optimization,
|
|
56
|
+
{
|
|
57
|
+
"max_turns": max_turns,
|
|
58
|
+
"candidates_per_turn": candidates,
|
|
59
|
+
"target": target,
|
|
60
|
+
"patience": patience,
|
|
61
|
+
"approval": approval,
|
|
62
|
+
"train_sample": train_sample,
|
|
63
|
+
"instructions": _instructions(project, instructions, feedback_file),
|
|
64
|
+
},
|
|
65
|
+
)
|
|
66
|
+
budgets = apply_overrides(
|
|
67
|
+
project.config.budgets,
|
|
68
|
+
{
|
|
69
|
+
"max_total_model_calls": max_calls,
|
|
70
|
+
"max_total_cost_usd": max_cost,
|
|
71
|
+
"max_wall_time_minutes": max_minutes,
|
|
72
|
+
},
|
|
73
|
+
)
|
|
74
|
+
|
|
75
|
+
def say(line: str) -> None:
|
|
76
|
+
typer.echo(line, err=as_json)
|
|
77
|
+
|
|
78
|
+
with project.database() as conn:
|
|
79
|
+
try:
|
|
80
|
+
summary = asyncio.run(
|
|
81
|
+
optimize(
|
|
82
|
+
project,
|
|
83
|
+
conn,
|
|
84
|
+
settings,
|
|
85
|
+
budgets,
|
|
86
|
+
resume=resume,
|
|
87
|
+
use_cache=not no_cache,
|
|
88
|
+
say=say,
|
|
89
|
+
)
|
|
90
|
+
)
|
|
91
|
+
except (KeyboardInterrupt, asyncio.CancelledError):
|
|
92
|
+
typer.echo(
|
|
93
|
+
"Interrupted. Progress is saved; continue with 'strop optimize --resume'.",
|
|
94
|
+
err=True,
|
|
95
|
+
)
|
|
96
|
+
raise typer.Exit(ExitCode.INTERRUPTED) from None
|
|
97
|
+
_show(summary, as_json)
|
|
98
|
+
if summary.stop_reason == "awaiting_approval":
|
|
99
|
+
_next_steps(summary)
|
|
100
|
+
raise typer.Exit(ExitCode.PAUSED)
|
|
101
|
+
if summary.stop_reason == "budget":
|
|
102
|
+
typer.echo("Raise the budget and continue with 'strop optimize --resume'.", err=True)
|
|
103
|
+
raise typer.Exit(ExitCode.BUDGET)
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
def approve_command(
|
|
107
|
+
version: Annotated[str, typer.Argument(help="Pending version, such as v4.")],
|
|
108
|
+
directory: DirOption = Path(),
|
|
109
|
+
) -> None:
|
|
110
|
+
"""Accept a pending candidate as the best prompt."""
|
|
111
|
+
project = open_project(directory)
|
|
112
|
+
with project.database() as conn:
|
|
113
|
+
candidate = approve(conn, version)
|
|
114
|
+
typer.echo(f"v{candidate.version_number} is now the best prompt.")
|
|
115
|
+
typer.echo("Continue with 'strop optimize --resume'.")
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
def reject_command(
|
|
119
|
+
version: Annotated[str, typer.Argument(help="Pending version, such as v4.")],
|
|
120
|
+
reason: Annotated[str, Option(help="Why it is rejected.")] = "",
|
|
121
|
+
directory: DirOption = Path(),
|
|
122
|
+
) -> None:
|
|
123
|
+
"""Discard a pending candidate. The best prompt stays unchanged."""
|
|
124
|
+
project = open_project(directory)
|
|
125
|
+
with project.database() as conn:
|
|
126
|
+
candidate = reject(conn, version, reason)
|
|
127
|
+
typer.echo(f"v{candidate.version_number} was rejected; the best prompt is unchanged.")
|
|
128
|
+
typer.echo("Continue with 'strop optimize --resume'.")
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
def _instructions(project: Project, text: str | None, file: Path | None) -> str | None:
|
|
132
|
+
parts = [project.config.optimization.instructions, text or ""]
|
|
133
|
+
if file is not None:
|
|
134
|
+
try:
|
|
135
|
+
parts.append(file.read_text(encoding="utf-8"))
|
|
136
|
+
except OSError as exc:
|
|
137
|
+
raise ConfigError(f"cannot read the feedback file {file}: {exc.strerror}") from exc
|
|
138
|
+
joined = "\n".join(part.strip() for part in parts if part.strip())
|
|
139
|
+
return joined or None
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
def _show(summary: RunSummary, as_json: bool) -> None:
|
|
143
|
+
if as_json:
|
|
144
|
+
emit_json(run_payload(summary))
|
|
145
|
+
return
|
|
146
|
+
render_run(make_console(ColorMode.AUTO), summary)
|
|
147
|
+
|
|
148
|
+
|
|
149
|
+
def _next_steps(summary: RunSummary) -> None:
|
|
150
|
+
pending = summary.pending
|
|
151
|
+
if pending is None or pending.version_number is None:
|
|
152
|
+
return
|
|
153
|
+
ref = f"v{pending.version_number}"
|
|
154
|
+
typer.echo(f"Review it with 'strop diff {ref}', then:", err=True)
|
|
155
|
+
typer.echo(f" strop approve {ref} or strop reject {ref}", err=True)
|
|
156
|
+
typer.echo(" strop optimize --resume", err=True)
|
strop/cli/options.py
ADDED
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
"""Command-line options shared by several commands."""
|
|
2
|
+
|
|
3
|
+
import json
|
|
4
|
+
from collections.abc import Mapping
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
from typing import Annotated, Any
|
|
7
|
+
|
|
8
|
+
import typer
|
|
9
|
+
|
|
10
|
+
DirOption = Annotated[Path, typer.Option("--dir", help="Project directory.")]
|
|
11
|
+
JsonOption = Annotated[bool, typer.Option("--json", help="Print machine-readable JSON.")]
|
|
12
|
+
SkipInvalidOption = Annotated[
|
|
13
|
+
bool,
|
|
14
|
+
typer.Option("--skip-invalid", help="Exclude invalid dataset rows instead of failing."),
|
|
15
|
+
]
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def emit_json(payload: Mapping[str, Any]) -> None:
|
|
19
|
+
"""Print ``payload`` as the command's single JSON document on stdout."""
|
|
20
|
+
typer.echo(json.dumps(payload, indent=2, ensure_ascii=False))
|
strop/cli/progress.py
ADDED
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
"""A progress line on stderr, shown only on an interactive terminal."""
|
|
2
|
+
|
|
3
|
+
import sys
|
|
4
|
+
|
|
5
|
+
from strop.evaluation.runner import Progress
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def stderr_progress(label: str) -> Progress | None:
|
|
9
|
+
"""Return a callback that rewrites ``label: done/total`` on stderr, or None when piped."""
|
|
10
|
+
if not sys.stderr.isatty():
|
|
11
|
+
return None
|
|
12
|
+
|
|
13
|
+
def update(done: int, total: int) -> None:
|
|
14
|
+
end = "\n" if done >= total else ""
|
|
15
|
+
sys.stderr.write(f"\r{label}: {done}/{total}{end}")
|
|
16
|
+
sys.stderr.flush()
|
|
17
|
+
|
|
18
|
+
return update
|