strop-cli 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (137) hide show
  1. strop/__init__.py +5 -0
  2. strop/__main__.py +5 -0
  3. strop/analysis/__init__.py +17 -0
  4. strop/analysis/classify.py +52 -0
  5. strop/analysis/evidence.py +52 -0
  6. strop/analysis/facts.py +74 -0
  7. strop/analysis/patterns.py +90 -0
  8. strop/cli/__init__.py +5 -0
  9. strop/cli/analyze.py +93 -0
  10. strop/cli/app.py +82 -0
  11. strop/cli/data.py +111 -0
  12. strop/cli/evaluate.py +145 -0
  13. strop/cli/export.py +74 -0
  14. strop/cli/final.py +106 -0
  15. strop/cli/init.py +131 -0
  16. strop/cli/optimize.py +156 -0
  17. strop/cli/options.py +20 -0
  18. strop/cli/progress.py +18 -0
  19. strop/cli/prompts.py +110 -0
  20. strop/cli/status.py +68 -0
  21. strop/cli/versions.py +123 -0
  22. strop/config/__init__.py +16 -0
  23. strop/config/base.py +9 -0
  24. strop/config/constraints.py +33 -0
  25. strop/config/dataset.py +63 -0
  26. strop/config/enums.py +52 -0
  27. strop/config/io.py +52 -0
  28. strop/config/limits.py +19 -0
  29. strop/config/llm.py +42 -0
  30. strop/config/metrics.py +86 -0
  31. strop/config/optimization.py +23 -0
  32. strop/config/output.py +31 -0
  33. strop/config/overrides.py +22 -0
  34. strop/config/root.py +36 -0
  35. strop/config/rules.py +67 -0
  36. strop/config/split.py +23 -0
  37. strop/data/__init__.py +58 -0
  38. strop/data/loaders.py +85 -0
  39. strop/data/models.py +44 -0
  40. strop/data/rows.py +99 -0
  41. strop/data/splitting.py +103 -0
  42. strop/env.py +44 -0
  43. strop/errors.py +59 -0
  44. strop/evaluation/__init__.py +1 -0
  45. strop/evaluation/budget.py +67 -0
  46. strop/evaluation/budgeted.py +23 -0
  47. strop/evaluation/examples.py +49 -0
  48. strop/evaluation/models.py +105 -0
  49. strop/evaluation/parsing.py +96 -0
  50. strop/evaluation/runner.py +107 -0
  51. strop/formatting.py +14 -0
  52. strop/llm/__init__.py +7 -0
  53. strop/llm/caching.py +28 -0
  54. strop/llm/client.py +15 -0
  55. strop/llm/factory.py +14 -0
  56. strop/llm/litellm_client.py +90 -0
  57. strop/llm/mock.py +43 -0
  58. strop/llm/retry.py +44 -0
  59. strop/llm/structured.py +52 -0
  60. strop/llm/types.py +59 -0
  61. strop/log.py +49 -0
  62. strop/metrics/__init__.py +1 -0
  63. strop/metrics/basic.py +30 -0
  64. strop/metrics/classification.py +164 -0
  65. strop/metrics/judge.py +130 -0
  66. strop/metrics/python_metric.py +101 -0
  67. strop/metrics/python_runner.py +65 -0
  68. strop/metrics/registry.py +115 -0
  69. strop/metrics/rules.py +55 -0
  70. strop/metrics/structured.py +109 -0
  71. strop/metrics/suite.py +109 -0
  72. strop/metrics/text.py +55 -0
  73. strop/metrics/types.py +106 -0
  74. strop/optimization/__init__.py +25 -0
  75. strop/optimization/acceptance.py +170 -0
  76. strop/optimization/generator.py +92 -0
  77. strop/optimization/meta_prompt.py +101 -0
  78. strop/optimization/types.py +69 -0
  79. strop/prompts/__init__.py +18 -0
  80. strop/prompts/diff.py +139 -0
  81. strop/prompts/files.py +92 -0
  82. strop/prompts/models.py +51 -0
  83. strop/prompts/render.py +16 -0
  84. strop/prompts/template.py +45 -0
  85. strop/prompts/validation.py +31 -0
  86. strop/prompts/versions.py +34 -0
  87. strop/py.typed +0 -0
  88. strop/reporting/__init__.py +6 -0
  89. strop/reporting/console.py +20 -0
  90. strop/reporting/diff_view.py +102 -0
  91. strop/reporting/evaluation_view.py +141 -0
  92. strop/reporting/final_view.py +109 -0
  93. strop/reporting/prompt_export.py +58 -0
  94. strop/reporting/report_markdown.py +157 -0
  95. strop/reporting/run_view.py +118 -0
  96. strop/services/__init__.py +1 -0
  97. strop/services/analysis.py +124 -0
  98. strop/services/clean.py +38 -0
  99. strop/services/compare.py +60 -0
  100. strop/services/context.py +70 -0
  101. strop/services/data.py +65 -0
  102. strop/services/evaluate.py +105 -0
  103. strop/services/evaluator.py +169 -0
  104. strop/services/export.py +90 -0
  105. strop/services/final_test.py +69 -0
  106. strop/services/metrics.py +89 -0
  107. strop/services/optimize.py +187 -0
  108. strop/services/partitions.py +46 -0
  109. strop/services/project.py +55 -0
  110. strop/services/prompts.py +76 -0
  111. strop/services/report.py +194 -0
  112. strop/services/review.py +43 -0
  113. strop/services/scaffold.py +79 -0
  114. strop/services/turn.py +234 -0
  115. strop/storage/__init__.py +6 -0
  116. strop/storage/cache.py +45 -0
  117. strop/storage/database.py +45 -0
  118. strop/storage/datasets.py +129 -0
  119. strop/storage/evaluations.py +160 -0
  120. strop/storage/migrations/001_initial.sql +5 -0
  121. strop/storage/migrations/002_datasets.sql +27 -0
  122. strop/storage/migrations/003_prompt_versions.sql +22 -0
  123. strop/storage/migrations/004_llm_cache.sql +6 -0
  124. strop/storage/migrations/005_evaluations.sql +37 -0
  125. strop/storage/migrations/006_runs.sql +52 -0
  126. strop/storage/migrations/007_example_feedback.sql +2 -0
  127. strop/storage/migrations/__init__.py +5 -0
  128. strop/storage/migrator.py +91 -0
  129. strop/storage/paths.py +30 -0
  130. strop/storage/prompts.py +96 -0
  131. strop/storage/run_models.py +75 -0
  132. strop/storage/runs.py +263 -0
  133. strop_cli-0.1.0.dist-info/METADATA +242 -0
  134. strop_cli-0.1.0.dist-info/RECORD +137 -0
  135. strop_cli-0.1.0.dist-info/WHEEL +4 -0
  136. strop_cli-0.1.0.dist-info/entry_points.txt +2 -0
  137. strop_cli-0.1.0.dist-info/licenses/LICENSE +21 -0
strop/cli/evaluate.py ADDED
@@ -0,0 +1,145 @@
1
+ """The ``baseline`` and ``eval`` commands."""
2
+
3
+ import asyncio
4
+ from enum import StrEnum
5
+ from pathlib import Path
6
+ from typing import Annotated
7
+
8
+ import typer
9
+
10
+ from strop.cli.options import DirOption, JsonOption, emit_json
11
+ from strop.cli.progress import stderr_progress
12
+ from strop.data.models import Partition
13
+ from strop.evaluation.models import EvaluationRecord
14
+ from strop.formatting import fmt
15
+ from strop.reporting import ColorMode, make_console
16
+ from strop.reporting.evaluation_view import record_payload, render_evaluation
17
+ from strop.services.evaluate import BaselineOutcome, run_baseline, run_eval
18
+ from strop.services.project import Project, open_project
19
+ from strop.services.prompts import has_unregistered_edits
20
+ from strop.storage.evaluations import EvaluationRepository
21
+ from strop.storage.prompts import PromptRepository
22
+
23
+ NoCacheOption = Annotated[bool, typer.Option("--no-cache", help="Do not read or write the cache.")]
24
+ SampleOption = Annotated[
25
+ int | None, typer.Option(help="Use a seeded random sample of this many rows.", min=1)
26
+ ]
27
+ ShowErrorsOption = Annotated[int, typer.Option(help="How many failing examples to list.", min=0)]
28
+
29
+
30
+ class EvalPartition(StrEnum):
31
+ """Partitions that may be evaluated freely. The test partition is not among them."""
32
+
33
+ TRAIN = "train"
34
+ VALIDATION = "validation"
35
+
36
+
37
+ def baseline_command(
38
+ repeats: Annotated[int, typer.Option(help="Run the baseline this many times.", min=1)] = 1,
39
+ sample: SampleOption = None,
40
+ with_train: Annotated[bool, typer.Option(help="Also evaluate on the train partition.")] = False,
41
+ no_cache: NoCacheOption = False,
42
+ show_errors: ShowErrorsOption = 5,
43
+ as_json: JsonOption = False,
44
+ directory: DirOption = Path(),
45
+ ) -> None:
46
+ """Measure the initial prompt (v1) on the validation partition."""
47
+ project = open_project(directory)
48
+ with project.database() as conn:
49
+ progress = None if as_json else stderr_progress("Baseline")
50
+ outcome = asyncio.run(
51
+ run_baseline(
52
+ project,
53
+ conn,
54
+ repeats=repeats,
55
+ sample=sample,
56
+ with_train=with_train,
57
+ use_cache=not no_cache,
58
+ progress=progress,
59
+ )
60
+ )
61
+ repo = EvaluationRepository(conn)
62
+ _warn_about_edits(project, PromptRepository(conn))
63
+ if as_json:
64
+ emit_json(_baseline_payload(outcome))
65
+ return
66
+ console = make_console(ColorMode.AUTO)
67
+ for record in [outcome.reference, *([outcome.train] if outcome.train else [])]:
68
+ render_evaluation(
69
+ console,
70
+ record,
71
+ repo.outcomes(record.id),
72
+ version_ref=outcome.version.ref,
73
+ show_errors=show_errors,
74
+ )
75
+ if outcome.noise is not None:
76
+ typer.echo(
77
+ f"Run-to-run noise over {repeats} runs: the primary metric varied by "
78
+ f"{fmt(outcome.noise)}. Improvements smaller than this are not meaningful."
79
+ )
80
+
81
+
82
+ def eval_command(
83
+ version: Annotated[
84
+ str | None, typer.Argument(help="Version such as v2; default: best.")
85
+ ] = None,
86
+ partition: Annotated[EvalPartition, typer.Option(help="Partition to evaluate.")] = (
87
+ EvalPartition.VALIDATION
88
+ ),
89
+ sample: SampleOption = None,
90
+ no_cache: NoCacheOption = False,
91
+ show_errors: ShowErrorsOption = 5,
92
+ as_json: JsonOption = False,
93
+ directory: DirOption = Path(),
94
+ ) -> None:
95
+ """Score a prompt version on a partition. Nothing is changed or promoted."""
96
+ project = open_project(directory)
97
+ with project.database() as conn:
98
+ selected, record = asyncio.run(
99
+ run_eval(
100
+ project,
101
+ conn,
102
+ ref=version,
103
+ partition=Partition(partition.value),
104
+ sample=sample,
105
+ use_cache=not no_cache,
106
+ progress=None if as_json else stderr_progress("Evaluating"),
107
+ )
108
+ )
109
+ outcomes = EvaluationRepository(conn).outcomes(record.id)
110
+ if version is None:
111
+ _warn_about_edits(project, PromptRepository(conn))
112
+ if as_json:
113
+ emit_json(record_payload(record, selected.ref))
114
+ return
115
+ render_evaluation(
116
+ make_console(ColorMode.AUTO),
117
+ record,
118
+ outcomes,
119
+ version_ref=selected.ref,
120
+ show_errors=show_errors,
121
+ )
122
+
123
+
124
+ def _warn_about_edits(project: Project, repo: PromptRepository) -> None:
125
+ if has_unregistered_edits(project, repo):
126
+ typer.echo(
127
+ f"Warning: {project.config.prompt.file} differs from every stored prompt version. "
128
+ f"Run 'strop prompt add --file {project.config.prompt.file} --use' to use your edits.",
129
+ err=True,
130
+ )
131
+
132
+
133
+ def _baseline_payload(outcome: BaselineOutcome) -> dict[str, object]:
134
+ ref = outcome.version.ref
135
+
136
+ def one(record: EvaluationRecord) -> dict[str, object]:
137
+ return dict(record_payload(record, ref))
138
+
139
+ return {
140
+ "version": ref,
141
+ "baseline": one(outcome.reference),
142
+ "repeats": [one(r) for r in outcome.records],
143
+ "noise": outcome.noise,
144
+ "train": one(outcome.train) if outcome.train else None,
145
+ }
strop/cli/export.py ADDED
@@ -0,0 +1,74 @@
1
+ """The ``export`` and ``clean`` commands."""
2
+
3
+ from pathlib import Path
4
+ from typing import Annotated
5
+
6
+ import typer
7
+
8
+ from strop.cli.options import DirOption, JsonOption, emit_json
9
+ from strop.reporting.prompt_export import PromptFormat
10
+ from strop.services.clean import clean_project
11
+ from strop.services.export import export_project
12
+ from strop.services.project import open_project
13
+
14
+
15
+ def export_command(
16
+ version: Annotated[
17
+ str | None, typer.Argument(help="Version such as v2; default: best.")
18
+ ] = None,
19
+ out: Annotated[Path, typer.Option(help="Output directory, relative to the project.")] = Path(
20
+ "exports"
21
+ ),
22
+ prompt_format: Annotated[
23
+ PromptFormat, typer.Option(help="Format of the exported prompt file.")
24
+ ] = PromptFormat.MD,
25
+ results: Annotated[
26
+ bool, typer.Option(help="Also write results.csv with per-example model outputs.")
27
+ ] = True,
28
+ write_prompt: Annotated[
29
+ Path | None, typer.Option(help="Also write the prompt as a Strop prompt YAML file here.")
30
+ ] = None,
31
+ force: Annotated[bool, typer.Option(help="Overwrite existing files.")] = False,
32
+ as_json: JsonOption = False,
33
+ directory: DirOption = Path(),
34
+ ) -> None:
35
+ """Export a prompt version, a JSON and Markdown report, and per-example results."""
36
+ project = open_project(directory)
37
+ with project.database() as conn:
38
+ result = export_project(
39
+ project,
40
+ conn,
41
+ ref=version,
42
+ out=directory / out,
43
+ prompt_format=prompt_format,
44
+ include_results=results,
45
+ write_prompt=directory / write_prompt if write_prompt else None,
46
+ force=force,
47
+ )
48
+ if as_json:
49
+ emit_json({"version": result.version.number, "files": [str(p) for p in result.files]})
50
+ return
51
+ typer.echo(f"Exported {result.version.ref}:")
52
+ for path in result.files:
53
+ typer.echo(f" {path}")
54
+ if results:
55
+ typer.echo("Note: results.csv contains model outputs for your dataset rows.")
56
+
57
+
58
+ def clean_command(
59
+ cache: Annotated[bool, typer.Option("--cache", help="Delete cached model responses.")] = False,
60
+ outputs: Annotated[
61
+ bool, typer.Option("--outputs", help="Delete stored model outputs but keep the scores.")
62
+ ] = False,
63
+ everything: Annotated[
64
+ bool, typer.Option("--all", help="Delete the whole .strop directory.")
65
+ ] = False,
66
+ yes: Annotated[bool, typer.Option("--yes", help="Confirm the deletion.")] = False,
67
+ directory: DirOption = Path(),
68
+ ) -> None:
69
+ """Delete stored data. strop.yaml, the prompt file, and the dataset are never touched."""
70
+ project = open_project(directory)
71
+ for line in clean_project(
72
+ project, cache=cache, outputs=outputs, everything=everything, confirmed=yes
73
+ ):
74
+ typer.echo(line)
strop/cli/final.py ADDED
@@ -0,0 +1,106 @@
1
+ """The ``test`` and ``compare`` commands."""
2
+
3
+ import asyncio
4
+ from pathlib import Path
5
+ from typing import Annotated
6
+
7
+ import typer
8
+
9
+ from strop.cli.evaluate import EvalPartition, NoCacheOption, SampleOption, ShowErrorsOption
10
+ from strop.cli.options import DirOption, JsonOption, emit_json
11
+ from strop.cli.progress import stderr_progress
12
+ from strop.data.models import Partition
13
+ from strop.reporting import ColorMode, make_console
14
+ from strop.reporting.evaluation_view import render_evaluation
15
+ from strop.reporting.final_view import (
16
+ comparison_payload,
17
+ final_test_payload,
18
+ render_comparison,
19
+ render_final_test,
20
+ )
21
+ from strop.services.compare import parse_models, run_compare
22
+ from strop.services.final_test import run_final_test
23
+ from strop.services.project import open_project
24
+ from strop.storage.evaluations import EvaluationRepository
25
+
26
+
27
+ def final_test_command(
28
+ version: Annotated[
29
+ str | None, typer.Argument(help="Version such as v2; default: best.")
30
+ ] = None,
31
+ confirm: Annotated[
32
+ bool, typer.Option("--confirm", help="Acknowledge that this uses up the test set.")
33
+ ] = False,
34
+ allow_rerun: Annotated[
35
+ bool, typer.Option("--allow-rerun", help="Allow a second test evaluation.")
36
+ ] = False,
37
+ with_baseline: Annotated[
38
+ bool, typer.Option("--with-baseline", help="Also test the initial prompt (v1).")
39
+ ] = False,
40
+ no_cache: NoCacheOption = False,
41
+ show_errors: ShowErrorsOption = 5,
42
+ as_json: JsonOption = False,
43
+ directory: DirOption = Path(),
44
+ ) -> None:
45
+ """Run the final evaluation on the held-out test partition.
46
+
47
+ The test set is only a fair estimate if it is looked at once. Pass --confirm to proceed.
48
+ """
49
+ project = open_project(directory)
50
+ with project.database() as conn:
51
+ result = asyncio.run(
52
+ run_final_test(
53
+ project,
54
+ conn,
55
+ ref=version,
56
+ confirm=confirm,
57
+ allow_rerun=allow_rerun,
58
+ with_baseline=with_baseline,
59
+ use_cache=not no_cache,
60
+ progress=None if as_json else stderr_progress("Testing"),
61
+ )
62
+ )
63
+ outcomes = EvaluationRepository(conn).outcomes(result.record.id)
64
+ if as_json:
65
+ emit_json(final_test_payload(result))
66
+ return
67
+ console = make_console(ColorMode.AUTO)
68
+ render_evaluation(
69
+ console, result.record, outcomes, version_ref=result.version.ref, show_errors=show_errors
70
+ )
71
+ render_final_test(console, result)
72
+
73
+
74
+ def compare_command(
75
+ models: Annotated[str, typer.Option(help="Comma-separated target models to compare.")],
76
+ version: Annotated[
77
+ str | None, typer.Argument(help="Version such as v2; default: best.")
78
+ ] = None,
79
+ partition: Annotated[EvalPartition, typer.Option(help="Partition to evaluate.")] = (
80
+ EvalPartition.VALIDATION
81
+ ),
82
+ sample: SampleOption = None,
83
+ no_cache: NoCacheOption = False,
84
+ as_json: JsonOption = False,
85
+ directory: DirOption = Path(),
86
+ ) -> None:
87
+ """Run one prompt version on several models and compare the results."""
88
+ names = parse_models(models)
89
+ project = open_project(directory)
90
+ with project.database() as conn:
91
+ comparison = asyncio.run(
92
+ run_compare(
93
+ project,
94
+ conn,
95
+ models=names,
96
+ ref=version,
97
+ partition=Partition(partition.value),
98
+ sample=sample,
99
+ use_cache=not no_cache,
100
+ progress=None if as_json else stderr_progress("Comparing"),
101
+ )
102
+ )
103
+ if as_json:
104
+ emit_json(comparison_payload(comparison))
105
+ return
106
+ render_comparison(make_console(ColorMode.AUTO), comparison)
strop/cli/init.py ADDED
@@ -0,0 +1,131 @@
1
+ """The ``strop init`` command."""
2
+
3
+ from pathlib import Path
4
+ from typing import Annotated, Any
5
+
6
+ import typer
7
+ from pydantic import ValidationError
8
+
9
+ from strop.cli.options import DirOption
10
+ from strop.config import DatasetFormat, OutputType, StropConfig
11
+ from strop.config.constraints import parse_constraints
12
+ from strop.config.io import format_errors
13
+ from strop.config.rules import load_rubrics, parse_rules
14
+ from strop.errors import ConfigError
15
+ from strop.services.scaffold import data_recipients, scaffold_project
16
+
17
+ Option = typer.Option
18
+
19
+
20
+ def init_command(
21
+ name: Annotated[str, Option(help="Project name.")],
22
+ task: Annotated[str, Option(help="One-sentence description of what the prompt must do.")],
23
+ data: Annotated[Path, Option(help="Dataset file (CSV or JSONL), relative to the project.")],
24
+ input_columns: Annotated[
25
+ list[str], Option("--input", help="Dataset column used as a prompt variable. Repeatable.")
26
+ ],
27
+ output_type: Annotated[OutputType, Option(help="Kind of output the model must produce.")],
28
+ metric: Annotated[str, Option(help="Primary metric, for example macro_f1.")],
29
+ target_model: Annotated[str, Option(help="Model the prompt is optimized for (LiteLLM name).")],
30
+ expected: Annotated[
31
+ str | None, Option(help="Dataset column holding the expected output.")
32
+ ] = None,
33
+ metadata: Annotated[
34
+ list[str] | None, Option(help="Dataset column kept as metadata. Repeatable.")
35
+ ] = None,
36
+ labels: Annotated[str, Option(help="Comma-separated labels, for output type 'label'.")] = "",
37
+ output_schema: Annotated[
38
+ Path | None, Option(help="JSON schema file the output must satisfy, for type 'json'.")
39
+ ] = None,
40
+ secondary_metric: Annotated[
41
+ list[str] | None, Option(help="Additional metric to report. Repeatable.")
42
+ ] = None,
43
+ constraint: Annotated[
44
+ list[str] | None, Option(help="Metric limit such as 'valid_output_rate>=0.98'. Repeatable.")
45
+ ] = None,
46
+ rule: Annotated[
47
+ list[str] | None, Option(help="Output check such as 'short=max_words:120'. Repeatable.")
48
+ ] = None,
49
+ metric_file: Annotated[
50
+ Path | None, Option(help="Python file defining score(output, row); metric name 'custom'.")
51
+ ] = None,
52
+ rubric_file: Annotated[
53
+ Path | None, Option(help="YAML file of judge rubrics; each becomes metric 'judge:<name>'.")
54
+ ] = None,
55
+ optimizer_model: Annotated[str | None, Option(help="Model that proposes revisions.")] = None,
56
+ analyzer_model: Annotated[str | None, Option(help="Model that summarizes failures.")] = None,
57
+ judge_model: Annotated[str | None, Option(help="Model that scores rubric metrics.")] = None,
58
+ prompt: Annotated[Path, Option(help="Prompt file to create or use.")] = Path("prompt.yaml"),
59
+ data_format: Annotated[
60
+ DatasetFormat | None, Option("--format", help="Dataset format; inferred if omitted.")
61
+ ] = None,
62
+ id_column: Annotated[str | None, Option(help="Dataset column with a unique row id.")] = None,
63
+ seed: Annotated[int, Option(help="Random seed for data splits.")] = 42,
64
+ directory: DirOption = Path(),
65
+ force: Annotated[bool, Option(help="Overwrite an existing strop.yaml.")] = False,
66
+ ) -> None:
67
+ """Create strop.yaml, a starter prompt, and the local .strop/ state directory."""
68
+ raw = {
69
+ "project": {"name": name, "task_description": task},
70
+ "prompt": {"file": str(prompt)},
71
+ "dataset": {
72
+ "path": str(data),
73
+ "format": data_format,
74
+ "id_column": id_column,
75
+ "columns": _column_roles(input_columns, expected, metadata or []),
76
+ },
77
+ "output": {
78
+ "type": output_type,
79
+ "labels": [item.strip() for item in labels.split(",") if item.strip()],
80
+ "schema_file": str(output_schema) if output_schema else None,
81
+ },
82
+ "models": _models(target_model, optimizer_model, analyzer_model, judge_model),
83
+ "metrics": {
84
+ "primary": metric,
85
+ "secondary": secondary_metric or [],
86
+ "constraints": {
87
+ k: v.model_dump() for k, v in parse_constraints(constraint or []).items()
88
+ },
89
+ "rules": {k: v.model_dump(mode="json") for k, v in parse_rules(rule or []).items()},
90
+ "python": {"file": str(metric_file)} if metric_file else None,
91
+ "judges": _rubrics(directory, rubric_file),
92
+ },
93
+ "split": {"seed": seed},
94
+ }
95
+ config = _validate(raw)
96
+ for entry in scaffold_project(config, directory, force=force):
97
+ typer.echo(f"Created {entry}")
98
+ typer.echo(f"Note: dataset rows will be sent to: {', '.join(data_recipients(config))}")
99
+ typer.echo("Note: keep .strop/ and .env out of version control; they hold outputs and keys.")
100
+
101
+
102
+ def _column_roles(inputs: list[str], expected: str | None, metadata: list[str]) -> dict[str, str]:
103
+ roles: dict[str, str] = {}
104
+ pairs = [(c, "input") for c in inputs] + [(c, "metadata") for c in metadata]
105
+ if expected:
106
+ pairs.append((expected, "expected"))
107
+ for column, role in pairs:
108
+ if column in roles:
109
+ raise ConfigError(f"column '{column}' is given more than one role")
110
+ roles[column] = role
111
+ return roles
112
+
113
+
114
+ def _models(
115
+ target: str, optimizer: str | None, analyzer: str | None, judge: str | None
116
+ ) -> dict[str, Any]:
117
+ chosen = {"target": target, "optimizer": optimizer, "analyzer": analyzer, "judge": judge}
118
+ return {role: {"model": model} for role, model in chosen.items() if model}
119
+
120
+
121
+ def _validate(raw: dict[str, Any]) -> StropConfig:
122
+ try:
123
+ return StropConfig.model_validate(raw)
124
+ except ValidationError as exc:
125
+ raise ConfigError(f"invalid options:\n{format_errors(exc)}") from exc
126
+
127
+
128
+ def _rubrics(directory: Path, file: Path | None) -> dict[str, Any]:
129
+ if file is None:
130
+ return {}
131
+ return {name: spec.model_dump() for name, spec in load_rubrics(directory / file).items()}
strop/cli/optimize.py ADDED
@@ -0,0 +1,156 @@
1
+ """The ``optimize``, ``approve`` and ``reject`` commands."""
2
+
3
+ import asyncio
4
+ from pathlib import Path
5
+ from typing import Annotated
6
+
7
+ import typer
8
+
9
+ from strop.cli.evaluate import NoCacheOption
10
+ from strop.cli.options import DirOption, JsonOption, emit_json
11
+ from strop.config.enums import ApprovalMode
12
+ from strop.config.overrides import apply_overrides
13
+ from strop.errors import ConfigError, ExitCode
14
+ from strop.reporting import ColorMode, make_console
15
+ from strop.reporting.run_view import render_run, run_payload
16
+ from strop.services.optimize import RunSummary, optimize
17
+ from strop.services.project import Project, open_project
18
+ from strop.services.review import approve, reject
19
+
20
+ Option = typer.Option
21
+
22
+
23
+ def optimize_command(
24
+ max_turns: Annotated[int | None, Option(help="Maximum number of turns.", min=1)] = None,
25
+ candidates: Annotated[int | None, Option(help="Candidates per turn.", min=1)] = None,
26
+ target: Annotated[
27
+ float | None, Option(help="Stop once the primary metric reaches this.")
28
+ ] = None,
29
+ patience: Annotated[
30
+ int | None, Option(help="Stop after this many turns without gain.", min=1)
31
+ ] = None,
32
+ approval: Annotated[
33
+ ApprovalMode | None, Option(help="auto, or manual to pause for approval.")
34
+ ] = None,
35
+ instructions: Annotated[str | None, Option(help="Extra guidance for the optimizer.")] = None,
36
+ feedback_file: Annotated[Path | None, Option(help="Text file with extra guidance.")] = None,
37
+ train_sample: Annotated[
38
+ int | None, Option(help="Analyze a sample of train rows.", min=1)
39
+ ] = None,
40
+ max_calls: Annotated[int | None, Option(help="Model call budget for this run.", min=1)] = None,
41
+ max_cost: Annotated[float | None, Option(help="Cost budget in USD.", min=0.0)] = None,
42
+ max_minutes: Annotated[float | None, Option(help="Time budget in minutes.", min=0.0)] = None,
43
+ optimizer_model: Annotated[str | None, Option(help="Use this model as the optimizer.")] = None,
44
+ analyzer_model: Annotated[str | None, Option(help="Use this model as the analyzer.")] = None,
45
+ resume: Annotated[bool, Option(help="Continue the unfinished run.")] = False,
46
+ no_cache: NoCacheOption = False,
47
+ as_json: JsonOption = False,
48
+ directory: DirOption = Path(),
49
+ ) -> None:
50
+ """Run the optimization loop: test the prompt, revise it from the failures, and repeat."""
51
+ project = open_project(directory).with_models(
52
+ {"optimizer": optimizer_model, "analyzer": analyzer_model}
53
+ )
54
+ settings = apply_overrides(
55
+ project.config.optimization,
56
+ {
57
+ "max_turns": max_turns,
58
+ "candidates_per_turn": candidates,
59
+ "target": target,
60
+ "patience": patience,
61
+ "approval": approval,
62
+ "train_sample": train_sample,
63
+ "instructions": _instructions(project, instructions, feedback_file),
64
+ },
65
+ )
66
+ budgets = apply_overrides(
67
+ project.config.budgets,
68
+ {
69
+ "max_total_model_calls": max_calls,
70
+ "max_total_cost_usd": max_cost,
71
+ "max_wall_time_minutes": max_minutes,
72
+ },
73
+ )
74
+
75
+ def say(line: str) -> None:
76
+ typer.echo(line, err=as_json)
77
+
78
+ with project.database() as conn:
79
+ try:
80
+ summary = asyncio.run(
81
+ optimize(
82
+ project,
83
+ conn,
84
+ settings,
85
+ budgets,
86
+ resume=resume,
87
+ use_cache=not no_cache,
88
+ say=say,
89
+ )
90
+ )
91
+ except (KeyboardInterrupt, asyncio.CancelledError):
92
+ typer.echo(
93
+ "Interrupted. Progress is saved; continue with 'strop optimize --resume'.",
94
+ err=True,
95
+ )
96
+ raise typer.Exit(ExitCode.INTERRUPTED) from None
97
+ _show(summary, as_json)
98
+ if summary.stop_reason == "awaiting_approval":
99
+ _next_steps(summary)
100
+ raise typer.Exit(ExitCode.PAUSED)
101
+ if summary.stop_reason == "budget":
102
+ typer.echo("Raise the budget and continue with 'strop optimize --resume'.", err=True)
103
+ raise typer.Exit(ExitCode.BUDGET)
104
+
105
+
106
+ def approve_command(
107
+ version: Annotated[str, typer.Argument(help="Pending version, such as v4.")],
108
+ directory: DirOption = Path(),
109
+ ) -> None:
110
+ """Accept a pending candidate as the best prompt."""
111
+ project = open_project(directory)
112
+ with project.database() as conn:
113
+ candidate = approve(conn, version)
114
+ typer.echo(f"v{candidate.version_number} is now the best prompt.")
115
+ typer.echo("Continue with 'strop optimize --resume'.")
116
+
117
+
118
+ def reject_command(
119
+ version: Annotated[str, typer.Argument(help="Pending version, such as v4.")],
120
+ reason: Annotated[str, Option(help="Why it is rejected.")] = "",
121
+ directory: DirOption = Path(),
122
+ ) -> None:
123
+ """Discard a pending candidate. The best prompt stays unchanged."""
124
+ project = open_project(directory)
125
+ with project.database() as conn:
126
+ candidate = reject(conn, version, reason)
127
+ typer.echo(f"v{candidate.version_number} was rejected; the best prompt is unchanged.")
128
+ typer.echo("Continue with 'strop optimize --resume'.")
129
+
130
+
131
+ def _instructions(project: Project, text: str | None, file: Path | None) -> str | None:
132
+ parts = [project.config.optimization.instructions, text or ""]
133
+ if file is not None:
134
+ try:
135
+ parts.append(file.read_text(encoding="utf-8"))
136
+ except OSError as exc:
137
+ raise ConfigError(f"cannot read the feedback file {file}: {exc.strerror}") from exc
138
+ joined = "\n".join(part.strip() for part in parts if part.strip())
139
+ return joined or None
140
+
141
+
142
+ def _show(summary: RunSummary, as_json: bool) -> None:
143
+ if as_json:
144
+ emit_json(run_payload(summary))
145
+ return
146
+ render_run(make_console(ColorMode.AUTO), summary)
147
+
148
+
149
+ def _next_steps(summary: RunSummary) -> None:
150
+ pending = summary.pending
151
+ if pending is None or pending.version_number is None:
152
+ return
153
+ ref = f"v{pending.version_number}"
154
+ typer.echo(f"Review it with 'strop diff {ref}', then:", err=True)
155
+ typer.echo(f" strop approve {ref} or strop reject {ref}", err=True)
156
+ typer.echo(" strop optimize --resume", err=True)
strop/cli/options.py ADDED
@@ -0,0 +1,20 @@
1
+ """Command-line options shared by several commands."""
2
+
3
+ import json
4
+ from collections.abc import Mapping
5
+ from pathlib import Path
6
+ from typing import Annotated, Any
7
+
8
+ import typer
9
+
10
+ DirOption = Annotated[Path, typer.Option("--dir", help="Project directory.")]
11
+ JsonOption = Annotated[bool, typer.Option("--json", help="Print machine-readable JSON.")]
12
+ SkipInvalidOption = Annotated[
13
+ bool,
14
+ typer.Option("--skip-invalid", help="Exclude invalid dataset rows instead of failing."),
15
+ ]
16
+
17
+
18
+ def emit_json(payload: Mapping[str, Any]) -> None:
19
+ """Print ``payload`` as the command's single JSON document on stdout."""
20
+ typer.echo(json.dumps(payload, indent=2, ensure_ascii=False))
strop/cli/progress.py ADDED
@@ -0,0 +1,18 @@
1
+ """A progress line on stderr, shown only on an interactive terminal."""
2
+
3
+ import sys
4
+
5
+ from strop.evaluation.runner import Progress
6
+
7
+
8
+ def stderr_progress(label: str) -> Progress | None:
9
+ """Return a callback that rewrites ``label: done/total`` on stderr, or None when piped."""
10
+ if not sys.stderr.isatty():
11
+ return None
12
+
13
+ def update(done: int, total: int) -> None:
14
+ end = "\n" if done >= total else ""
15
+ sys.stderr.write(f"\r{label}: {done}/{total}{end}")
16
+ sys.stderr.flush()
17
+
18
+ return update