strop-cli 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- strop/__init__.py +5 -0
- strop/__main__.py +5 -0
- strop/analysis/__init__.py +17 -0
- strop/analysis/classify.py +52 -0
- strop/analysis/evidence.py +52 -0
- strop/analysis/facts.py +74 -0
- strop/analysis/patterns.py +90 -0
- strop/cli/__init__.py +5 -0
- strop/cli/analyze.py +93 -0
- strop/cli/app.py +82 -0
- strop/cli/data.py +111 -0
- strop/cli/evaluate.py +145 -0
- strop/cli/export.py +74 -0
- strop/cli/final.py +106 -0
- strop/cli/init.py +131 -0
- strop/cli/optimize.py +156 -0
- strop/cli/options.py +20 -0
- strop/cli/progress.py +18 -0
- strop/cli/prompts.py +110 -0
- strop/cli/status.py +68 -0
- strop/cli/versions.py +123 -0
- strop/config/__init__.py +16 -0
- strop/config/base.py +9 -0
- strop/config/constraints.py +33 -0
- strop/config/dataset.py +63 -0
- strop/config/enums.py +52 -0
- strop/config/io.py +52 -0
- strop/config/limits.py +19 -0
- strop/config/llm.py +42 -0
- strop/config/metrics.py +86 -0
- strop/config/optimization.py +23 -0
- strop/config/output.py +31 -0
- strop/config/overrides.py +22 -0
- strop/config/root.py +36 -0
- strop/config/rules.py +67 -0
- strop/config/split.py +23 -0
- strop/data/__init__.py +58 -0
- strop/data/loaders.py +85 -0
- strop/data/models.py +44 -0
- strop/data/rows.py +99 -0
- strop/data/splitting.py +103 -0
- strop/env.py +44 -0
- strop/errors.py +59 -0
- strop/evaluation/__init__.py +1 -0
- strop/evaluation/budget.py +67 -0
- strop/evaluation/budgeted.py +23 -0
- strop/evaluation/examples.py +49 -0
- strop/evaluation/models.py +105 -0
- strop/evaluation/parsing.py +96 -0
- strop/evaluation/runner.py +107 -0
- strop/formatting.py +14 -0
- strop/llm/__init__.py +7 -0
- strop/llm/caching.py +28 -0
- strop/llm/client.py +15 -0
- strop/llm/factory.py +14 -0
- strop/llm/litellm_client.py +90 -0
- strop/llm/mock.py +43 -0
- strop/llm/retry.py +44 -0
- strop/llm/structured.py +52 -0
- strop/llm/types.py +59 -0
- strop/log.py +49 -0
- strop/metrics/__init__.py +1 -0
- strop/metrics/basic.py +30 -0
- strop/metrics/classification.py +164 -0
- strop/metrics/judge.py +130 -0
- strop/metrics/python_metric.py +101 -0
- strop/metrics/python_runner.py +65 -0
- strop/metrics/registry.py +115 -0
- strop/metrics/rules.py +55 -0
- strop/metrics/structured.py +109 -0
- strop/metrics/suite.py +109 -0
- strop/metrics/text.py +55 -0
- strop/metrics/types.py +106 -0
- strop/optimization/__init__.py +25 -0
- strop/optimization/acceptance.py +170 -0
- strop/optimization/generator.py +92 -0
- strop/optimization/meta_prompt.py +101 -0
- strop/optimization/types.py +69 -0
- strop/prompts/__init__.py +18 -0
- strop/prompts/diff.py +139 -0
- strop/prompts/files.py +92 -0
- strop/prompts/models.py +51 -0
- strop/prompts/render.py +16 -0
- strop/prompts/template.py +45 -0
- strop/prompts/validation.py +31 -0
- strop/prompts/versions.py +34 -0
- strop/py.typed +0 -0
- strop/reporting/__init__.py +6 -0
- strop/reporting/console.py +20 -0
- strop/reporting/diff_view.py +102 -0
- strop/reporting/evaluation_view.py +141 -0
- strop/reporting/final_view.py +109 -0
- strop/reporting/prompt_export.py +58 -0
- strop/reporting/report_markdown.py +157 -0
- strop/reporting/run_view.py +118 -0
- strop/services/__init__.py +1 -0
- strop/services/analysis.py +124 -0
- strop/services/clean.py +38 -0
- strop/services/compare.py +60 -0
- strop/services/context.py +70 -0
- strop/services/data.py +65 -0
- strop/services/evaluate.py +105 -0
- strop/services/evaluator.py +169 -0
- strop/services/export.py +90 -0
- strop/services/final_test.py +69 -0
- strop/services/metrics.py +89 -0
- strop/services/optimize.py +187 -0
- strop/services/partitions.py +46 -0
- strop/services/project.py +55 -0
- strop/services/prompts.py +76 -0
- strop/services/report.py +194 -0
- strop/services/review.py +43 -0
- strop/services/scaffold.py +79 -0
- strop/services/turn.py +234 -0
- strop/storage/__init__.py +6 -0
- strop/storage/cache.py +45 -0
- strop/storage/database.py +45 -0
- strop/storage/datasets.py +129 -0
- strop/storage/evaluations.py +160 -0
- strop/storage/migrations/001_initial.sql +5 -0
- strop/storage/migrations/002_datasets.sql +27 -0
- strop/storage/migrations/003_prompt_versions.sql +22 -0
- strop/storage/migrations/004_llm_cache.sql +6 -0
- strop/storage/migrations/005_evaluations.sql +37 -0
- strop/storage/migrations/006_runs.sql +52 -0
- strop/storage/migrations/007_example_feedback.sql +2 -0
- strop/storage/migrations/__init__.py +5 -0
- strop/storage/migrator.py +91 -0
- strop/storage/paths.py +30 -0
- strop/storage/prompts.py +96 -0
- strop/storage/run_models.py +75 -0
- strop/storage/runs.py +263 -0
- strop_cli-0.1.0.dist-info/METADATA +242 -0
- strop_cli-0.1.0.dist-info/RECORD +137 -0
- strop_cli-0.1.0.dist-info/WHEEL +4 -0
- strop_cli-0.1.0.dist-info/entry_points.txt +2 -0
- strop_cli-0.1.0.dist-info/licenses/LICENSE +21 -0
strop/cli/prompts.py
ADDED
|
@@ -0,0 +1,110 @@
|
|
|
1
|
+
"""The ``strop prompt`` command group."""
|
|
2
|
+
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
from typing import Annotated
|
|
5
|
+
|
|
6
|
+
import typer
|
|
7
|
+
|
|
8
|
+
from strop.cli.options import DirOption, JsonOption, emit_json
|
|
9
|
+
from strop.prompts.versions import PromptVersion
|
|
10
|
+
from strop.services.project import Project, open_project
|
|
11
|
+
from strop.services.prompts import add_version_from_file, load_project_prompt, resolve_version
|
|
12
|
+
from strop.storage.prompts import PromptRepository
|
|
13
|
+
|
|
14
|
+
prompt_app = typer.Typer(help="Add and inspect prompt versions.", no_args_is_help=True)
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
@prompt_app.command("add")
|
|
18
|
+
def add_command(
|
|
19
|
+
file: Annotated[Path, typer.Option(help="Prompt file holding the edited prompt.")],
|
|
20
|
+
parent: Annotated[
|
|
21
|
+
str | None, typer.Option(help="Version it derives from; default: best.")
|
|
22
|
+
] = None,
|
|
23
|
+
reason: Annotated[str, typer.Option(help="Why this edit was made.")] = "",
|
|
24
|
+
use: Annotated[bool, typer.Option("--use", help="Make this version the best prompt.")] = False,
|
|
25
|
+
directory: DirOption = Path(),
|
|
26
|
+
) -> None:
|
|
27
|
+
"""Save an edited prompt file as a new immutable version."""
|
|
28
|
+
project = open_project(directory)
|
|
29
|
+
with project.database() as conn:
|
|
30
|
+
repo = PromptRepository(conn)
|
|
31
|
+
version = add_version_from_file(project, repo, file, parent=parent, reason=reason, use=use)
|
|
32
|
+
typer.echo(f"Saved {version.ref} (parent v{version.parent_number})")
|
|
33
|
+
if use:
|
|
34
|
+
typer.echo(f"{version.ref} is now the best prompt.")
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
@prompt_app.command("show")
|
|
38
|
+
def show_command(
|
|
39
|
+
version: Annotated[
|
|
40
|
+
str | None, typer.Argument(help="Version such as v2; default: best.")
|
|
41
|
+
] = None,
|
|
42
|
+
as_json: JsonOption = False,
|
|
43
|
+
directory: DirOption = Path(),
|
|
44
|
+
) -> None:
|
|
45
|
+
"""Print the messages of a prompt version."""
|
|
46
|
+
project = open_project(directory)
|
|
47
|
+
with project.database() as conn:
|
|
48
|
+
repo = PromptRepository(conn)
|
|
49
|
+
if version is None and repo.first() is None:
|
|
50
|
+
_show_unregistered(project, as_json)
|
|
51
|
+
return
|
|
52
|
+
selected = resolve_version(repo, version)
|
|
53
|
+
is_best = repo.best_number() == selected.number
|
|
54
|
+
if as_json:
|
|
55
|
+
emit_json(_payload(selected, is_best=is_best))
|
|
56
|
+
return
|
|
57
|
+
typer.echo(_header(selected, is_best=is_best))
|
|
58
|
+
for message in selected.template.messages:
|
|
59
|
+
editable = "editable" if selected.template.is_editable(message) else "frozen"
|
|
60
|
+
typer.echo(f"--- {message.role} ({editable})")
|
|
61
|
+
typer.echo(message.content.rstrip("\n"))
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def _show_unregistered(project: Project, as_json: bool) -> None:
|
|
65
|
+
"""Before the first run there are no versions yet, so show the prompt file itself."""
|
|
66
|
+
template = load_project_prompt(project)
|
|
67
|
+
file = project.config.prompt.file
|
|
68
|
+
if as_json:
|
|
69
|
+
emit_json(
|
|
70
|
+
{
|
|
71
|
+
"version": None,
|
|
72
|
+
"file": str(file),
|
|
73
|
+
"messages": [
|
|
74
|
+
{"role": m.role, "content": m.content, "editable": template.is_editable(m)}
|
|
75
|
+
for m in template.messages
|
|
76
|
+
],
|
|
77
|
+
}
|
|
78
|
+
)
|
|
79
|
+
return
|
|
80
|
+
typer.echo(f"{file} (not registered yet; it becomes v1 on the first run)")
|
|
81
|
+
for message in template.messages:
|
|
82
|
+
editable = "editable" if template.is_editable(message) else "frozen"
|
|
83
|
+
typer.echo(f"--- {message.role} ({editable})")
|
|
84
|
+
typer.echo(message.content.rstrip("\n"))
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def _header(version: PromptVersion, *, is_best: bool) -> str:
|
|
88
|
+
parent = f"parent v{version.parent_number}" if version.parent_number else "no parent"
|
|
89
|
+
best = ", best" if is_best else ""
|
|
90
|
+
summary = f"\nSummary: {version.change_summary}" if version.change_summary else ""
|
|
91
|
+
return f"{version.ref} ({parent}, {version.source}{best}){summary}"
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def _payload(version: PromptVersion, *, is_best: bool) -> dict[str, object]:
|
|
95
|
+
return {
|
|
96
|
+
"version": version.number,
|
|
97
|
+
"parent": version.parent_number,
|
|
98
|
+
"source": version.source,
|
|
99
|
+
"best": is_best,
|
|
100
|
+
"change_summary": version.change_summary,
|
|
101
|
+
"reason": version.reason,
|
|
102
|
+
"messages": [
|
|
103
|
+
{
|
|
104
|
+
"role": m.role,
|
|
105
|
+
"content": m.content,
|
|
106
|
+
"editable": version.template.is_editable(m),
|
|
107
|
+
}
|
|
108
|
+
for m in version.template.messages
|
|
109
|
+
],
|
|
110
|
+
}
|
strop/cli/status.py
ADDED
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
"""The ``strop status`` command."""
|
|
2
|
+
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
|
|
5
|
+
import typer
|
|
6
|
+
|
|
7
|
+
from strop.cli.options import DirOption, JsonOption, emit_json
|
|
8
|
+
from strop.formatting import fmt
|
|
9
|
+
from strop.services.project import open_project
|
|
10
|
+
from strop.storage.datasets import DatasetRepository
|
|
11
|
+
from strop.storage.evaluations import EvaluationRepository
|
|
12
|
+
from strop.storage.prompts import PromptRepository
|
|
13
|
+
|
|
14
|
+
RECENT = 5
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def status_command(as_json: JsonOption = False, directory: DirOption = Path()) -> None:
|
|
18
|
+
"""Show the dataset split, the best prompt, and the latest evaluations."""
|
|
19
|
+
project = open_project(directory)
|
|
20
|
+
with project.database() as conn:
|
|
21
|
+
datasets = DatasetRepository(conn)
|
|
22
|
+
split = datasets.latest_split()
|
|
23
|
+
counts = datasets.partition_counts(split.id) if split else {}
|
|
24
|
+
prompts = PromptRepository(conn)
|
|
25
|
+
versions = prompts.list()
|
|
26
|
+
best = prompts.best_number()
|
|
27
|
+
recent = EvaluationRepository(conn).find(limit=RECENT)
|
|
28
|
+
payload = {
|
|
29
|
+
"project": project.config.project.name,
|
|
30
|
+
"split": (
|
|
31
|
+
{"id": split.id, "seed": split.seed, "counts": {p.value: n for p, n in counts.items()}}
|
|
32
|
+
if split
|
|
33
|
+
else None
|
|
34
|
+
),
|
|
35
|
+
"prompt_versions": len(versions),
|
|
36
|
+
"best_version": best,
|
|
37
|
+
"recent_evaluations": [
|
|
38
|
+
{
|
|
39
|
+
"id": e.id,
|
|
40
|
+
"version": e.prompt_number,
|
|
41
|
+
"partition": e.partition.value,
|
|
42
|
+
"kind": e.kind,
|
|
43
|
+
"primary_metric": e.report.primary,
|
|
44
|
+
"value": e.report.primary_value.value,
|
|
45
|
+
"samples": e.sample_count,
|
|
46
|
+
}
|
|
47
|
+
for e in recent
|
|
48
|
+
],
|
|
49
|
+
}
|
|
50
|
+
if as_json:
|
|
51
|
+
emit_json(payload)
|
|
52
|
+
return
|
|
53
|
+
typer.echo(f"Project: {project.config.project.name}")
|
|
54
|
+
if split:
|
|
55
|
+
sizes = ", ".join(f"{p.value} {n}" for p, n in counts.items())
|
|
56
|
+
typer.echo(f"Split {split.id} (seed {split.seed}): {sizes}")
|
|
57
|
+
else:
|
|
58
|
+
typer.echo("Split: none yet (run 'strop data split')")
|
|
59
|
+
if versions:
|
|
60
|
+
typer.echo(f"Prompt: v{best} is the best of {len(versions)} version(s)")
|
|
61
|
+
else:
|
|
62
|
+
typer.echo("Prompt: no versions yet (registered on the first run)")
|
|
63
|
+
for evaluation in recent:
|
|
64
|
+
typer.echo(
|
|
65
|
+
f" #{evaluation.id} v{evaluation.prompt_number} {evaluation.partition.value} "
|
|
66
|
+
f"{evaluation.kind}: {evaluation.report.primary}="
|
|
67
|
+
f"{fmt(evaluation.report.primary_value.value)} ({evaluation.sample_count} examples)"
|
|
68
|
+
)
|
strop/cli/versions.py
ADDED
|
@@ -0,0 +1,123 @@
|
|
|
1
|
+
"""Top-level commands about prompt history: ``diff``, ``history``, ``rollback``."""
|
|
2
|
+
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
from typing import Annotated
|
|
5
|
+
|
|
6
|
+
import typer
|
|
7
|
+
|
|
8
|
+
from strop.cli.options import DirOption, JsonOption, emit_json
|
|
9
|
+
from strop.errors import ConfigError
|
|
10
|
+
from strop.prompts.diff import diff_templates
|
|
11
|
+
from strop.prompts.versions import PromptVersion
|
|
12
|
+
from strop.reporting import ColorMode, make_console, render_diff
|
|
13
|
+
from strop.services.project import open_project
|
|
14
|
+
from strop.services.prompts import resolve_version
|
|
15
|
+
from strop.storage.prompts import PromptRepository
|
|
16
|
+
from strop.storage.runs import CandidateRecord, RunRepository
|
|
17
|
+
|
|
18
|
+
SUMMARY_WIDTH = 60
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def diff_command(
|
|
22
|
+
old: Annotated[str | None, typer.Argument(help="Older version, such as v1.")] = None,
|
|
23
|
+
new: Annotated[str | None, typer.Argument(help="Newer version; default: best.")] = None,
|
|
24
|
+
color: Annotated[ColorMode, typer.Option(help="When to use color.")] = ColorMode.AUTO,
|
|
25
|
+
as_json: JsonOption = False,
|
|
26
|
+
directory: DirOption = Path(),
|
|
27
|
+
) -> None:
|
|
28
|
+
"""Show a colored diff of two prompt versions.
|
|
29
|
+
|
|
30
|
+
With no arguments: the first prompt (v1) against the best prompt. With one argument:
|
|
31
|
+
that version against its parent.
|
|
32
|
+
"""
|
|
33
|
+
project = open_project(directory)
|
|
34
|
+
with project.database() as conn:
|
|
35
|
+
repo = PromptRepository(conn)
|
|
36
|
+
before, after = _pick_pair(repo, old, new)
|
|
37
|
+
diff = diff_templates(before.template, after.template)
|
|
38
|
+
if as_json:
|
|
39
|
+
emit_json({"from": before.ref, "to": after.ref, **diff.to_payload()})
|
|
40
|
+
return
|
|
41
|
+
console = make_console(color)
|
|
42
|
+
console.print(f"{before.ref} -> {after.ref}", soft_wrap=True, highlight=False)
|
|
43
|
+
render_diff(diff, console)
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def history_command(
|
|
47
|
+
as_json: JsonOption = False,
|
|
48
|
+
directory: DirOption = Path(),
|
|
49
|
+
) -> None:
|
|
50
|
+
"""List every prompt version with the decision made about it. The best is marked with *."""
|
|
51
|
+
project = open_project(directory)
|
|
52
|
+
with project.database() as conn:
|
|
53
|
+
repo = PromptRepository(conn)
|
|
54
|
+
runs = RunRepository(conn)
|
|
55
|
+
versions = repo.list()
|
|
56
|
+
best = repo.best_number()
|
|
57
|
+
decisions = {v.number: runs.for_version(v.number) for v in versions}
|
|
58
|
+
if as_json:
|
|
59
|
+
emit_json({"best": best, "versions": [_row(v, decisions[v.number]) for v in versions]})
|
|
60
|
+
return
|
|
61
|
+
if not versions:
|
|
62
|
+
typer.echo("No prompt versions yet. The prompt file is registered as v1 on the first run.")
|
|
63
|
+
return
|
|
64
|
+
for version in versions:
|
|
65
|
+
marker = "*" if version.number == best else " "
|
|
66
|
+
parent = f"v{version.parent_number}" if version.parent_number else "-"
|
|
67
|
+
decision = decisions[version.number]
|
|
68
|
+
status = decision.status if decision else "-"
|
|
69
|
+
delta = (
|
|
70
|
+
f"{decision.primary_delta:+.4f}"
|
|
71
|
+
if decision and decision.primary_delta is not None
|
|
72
|
+
else "-"
|
|
73
|
+
)
|
|
74
|
+
summary = version.change_summary[:SUMMARY_WIDTH]
|
|
75
|
+
typer.echo(
|
|
76
|
+
f"{marker} {version.ref:<5} {version.source:<10} parent {parent:<5} "
|
|
77
|
+
f"{status:<9} {delta:<8} {summary}"
|
|
78
|
+
)
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def rollback_command(
|
|
82
|
+
version: Annotated[str, typer.Argument(help="Version to make the best prompt.")],
|
|
83
|
+
directory: DirOption = Path(),
|
|
84
|
+
) -> None:
|
|
85
|
+
"""Make an earlier version the best prompt. No version is deleted or changed."""
|
|
86
|
+
project = open_project(directory)
|
|
87
|
+
with project.database() as conn:
|
|
88
|
+
repo = PromptRepository(conn)
|
|
89
|
+
target = resolve_version(repo, version)
|
|
90
|
+
previous = repo.best_number()
|
|
91
|
+
repo.set_best(target.number)
|
|
92
|
+
was = f" (was v{previous})" if previous else ""
|
|
93
|
+
typer.echo(f"Best prompt is now {target.ref}{was}. No versions were deleted.")
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def _pick_pair(
|
|
97
|
+
repo: PromptRepository, old: str | None, new: str | None
|
|
98
|
+
) -> tuple[PromptVersion, PromptVersion]:
|
|
99
|
+
if old is None:
|
|
100
|
+
first = repo.first()
|
|
101
|
+
if first is None:
|
|
102
|
+
raise ConfigError("no prompt versions yet; run 'strop baseline' first")
|
|
103
|
+
return first, resolve_version(repo, None)
|
|
104
|
+
if new is not None:
|
|
105
|
+
return resolve_version(repo, old), resolve_version(repo, new)
|
|
106
|
+
selected = resolve_version(repo, old)
|
|
107
|
+
if selected.parent_number is None:
|
|
108
|
+
raise ConfigError(f"{selected.ref} has no parent; name two versions to compare")
|
|
109
|
+
return repo.get(selected.parent_number), selected
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
def _row(version: PromptVersion, decision: CandidateRecord | None) -> dict[str, object]:
|
|
113
|
+
return {
|
|
114
|
+
"status": decision.status if decision else None,
|
|
115
|
+
"primary_delta": decision.primary_delta if decision else None,
|
|
116
|
+
"decision_reason": decision.reason if decision else None,
|
|
117
|
+
"version": version.number,
|
|
118
|
+
"parent": version.parent_number,
|
|
119
|
+
"source": version.source,
|
|
120
|
+
"change_summary": version.change_summary,
|
|
121
|
+
"reason": version.reason,
|
|
122
|
+
"created_at": version.created_at,
|
|
123
|
+
}
|
strop/config/__init__.py
ADDED
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
"""Typed project configuration and its YAML loader."""
|
|
2
|
+
|
|
3
|
+
from strop.config.enums import ApprovalMode, ColumnRole, DatasetFormat, ModelRole, OutputType
|
|
4
|
+
from strop.config.io import dump_config, load_config
|
|
5
|
+
from strop.config.root import StropConfig
|
|
6
|
+
|
|
7
|
+
__all__ = [
|
|
8
|
+
"ApprovalMode",
|
|
9
|
+
"ColumnRole",
|
|
10
|
+
"DatasetFormat",
|
|
11
|
+
"ModelRole",
|
|
12
|
+
"OutputType",
|
|
13
|
+
"StropConfig",
|
|
14
|
+
"dump_config",
|
|
15
|
+
"load_config",
|
|
16
|
+
]
|
strop/config/base.py
ADDED
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
"""Shared base class for configuration models."""
|
|
2
|
+
|
|
3
|
+
from pydantic import BaseModel, ConfigDict
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
class StrictModel(BaseModel):
|
|
7
|
+
"""Immutable model that rejects unknown keys, so typos in strop.yaml fail loudly."""
|
|
8
|
+
|
|
9
|
+
model_config = ConfigDict(extra="forbid", frozen=True)
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
"""Parsing of ``--constraint`` flags such as ``valid_output_rate>=0.98``."""
|
|
2
|
+
|
|
3
|
+
import re
|
|
4
|
+
from collections.abc import Iterable
|
|
5
|
+
|
|
6
|
+
from strop.config.metrics import Bound
|
|
7
|
+
from strop.errors import ConfigError
|
|
8
|
+
|
|
9
|
+
_EXPRESSION = re.compile(r"^\s*(?P<metric>[\w:.\-]+)\s*(?P<op>>=|<=)\s*(?P<value>[-+0-9.eE]+)\s*$")
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def parse_constraints(expressions: Iterable[str]) -> dict[str, Bound]:
|
|
13
|
+
"""Turn ``metric>=x`` and ``metric<=y`` expressions into per-metric bounds.
|
|
14
|
+
|
|
15
|
+
Two expressions for the same metric (one ``>=``, one ``<=``) are merged.
|
|
16
|
+
"""
|
|
17
|
+
limits: dict[str, dict[str, float]] = {}
|
|
18
|
+
for text in expressions:
|
|
19
|
+
match = _EXPRESSION.match(text)
|
|
20
|
+
if match is None:
|
|
21
|
+
raise ConfigError(
|
|
22
|
+
f"invalid constraint '{text}'; use the form 'metric>=0.9' or 'metric<=0.1'"
|
|
23
|
+
)
|
|
24
|
+
try:
|
|
25
|
+
value = float(match["value"])
|
|
26
|
+
except ValueError:
|
|
27
|
+
raise ConfigError(f"invalid number in constraint '{text}'") from None
|
|
28
|
+
side = "min" if match["op"] == ">=" else "max"
|
|
29
|
+
bounds = limits.setdefault(match["metric"], {})
|
|
30
|
+
if side in bounds:
|
|
31
|
+
raise ConfigError(f"constraint on '{match['metric']}' sets '{side}' more than once")
|
|
32
|
+
bounds[side] = value
|
|
33
|
+
return {metric: Bound(**bounds) for metric, bounds in limits.items()}
|
strop/config/dataset.py
ADDED
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
"""Dataset location and column roles."""
|
|
2
|
+
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
from typing import Self
|
|
5
|
+
|
|
6
|
+
from pydantic import model_validator
|
|
7
|
+
|
|
8
|
+
from strop.config.base import StrictModel
|
|
9
|
+
from strop.config.enums import ColumnRole, DatasetFormat
|
|
10
|
+
from strop.errors import ConfigError
|
|
11
|
+
|
|
12
|
+
_SUFFIX_FORMATS = {
|
|
13
|
+
".csv": DatasetFormat.CSV,
|
|
14
|
+
".jsonl": DatasetFormat.JSONL,
|
|
15
|
+
".ndjson": DatasetFormat.JSONL,
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
class DatasetConfig(StrictModel):
|
|
20
|
+
path: Path
|
|
21
|
+
format: DatasetFormat | None = None
|
|
22
|
+
id_column: str | None = None
|
|
23
|
+
columns: dict[str, ColumnRole]
|
|
24
|
+
|
|
25
|
+
@model_validator(mode="after")
|
|
26
|
+
def _check_roles(self) -> Self:
|
|
27
|
+
roles = list(self.columns.values())
|
|
28
|
+
if ColumnRole.INPUT not in roles:
|
|
29
|
+
raise ValueError("at least one column must have the role 'input'")
|
|
30
|
+
if roles.count(ColumnRole.EXPECTED) > 1:
|
|
31
|
+
raise ValueError("at most one column may have the role 'expected'")
|
|
32
|
+
for name in self._columns_with(ColumnRole.INPUT):
|
|
33
|
+
if not name.isidentifier():
|
|
34
|
+
raise ValueError(
|
|
35
|
+
f"input column '{name}' cannot be used as a prompt variable; "
|
|
36
|
+
"rename it to letters, digits, and underscores, not starting with a digit"
|
|
37
|
+
)
|
|
38
|
+
return self
|
|
39
|
+
|
|
40
|
+
@property
|
|
41
|
+
def resolved_format(self) -> DatasetFormat:
|
|
42
|
+
"""The explicit format, or the one implied by the file extension."""
|
|
43
|
+
if self.format is not None:
|
|
44
|
+
return self.format
|
|
45
|
+
try:
|
|
46
|
+
return _SUFFIX_FORMATS[self.path.suffix.lower()]
|
|
47
|
+
except KeyError:
|
|
48
|
+
raise ConfigError(
|
|
49
|
+
f"cannot infer the dataset format from '{self.path.name}'; "
|
|
50
|
+
"set dataset.format to csv or jsonl"
|
|
51
|
+
) from None
|
|
52
|
+
|
|
53
|
+
@property
|
|
54
|
+
def input_columns(self) -> tuple[str, ...]:
|
|
55
|
+
return self._columns_with(ColumnRole.INPUT)
|
|
56
|
+
|
|
57
|
+
@property
|
|
58
|
+
def expected_column(self) -> str | None:
|
|
59
|
+
expected = self._columns_with(ColumnRole.EXPECTED)
|
|
60
|
+
return expected[0] if expected else None
|
|
61
|
+
|
|
62
|
+
def _columns_with(self, role: ColumnRole) -> tuple[str, ...]:
|
|
63
|
+
return tuple(name for name, value in self.columns.items() if value == role)
|
strop/config/enums.py
ADDED
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
"""Enumerations used across the configuration sections."""
|
|
2
|
+
|
|
3
|
+
from enum import StrEnum
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
class DatasetFormat(StrEnum):
|
|
7
|
+
CSV = "csv"
|
|
8
|
+
JSONL = "jsonl"
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
class ColumnRole(StrEnum):
|
|
12
|
+
"""What a dataset column is used for."""
|
|
13
|
+
|
|
14
|
+
INPUT = "input"
|
|
15
|
+
EXPECTED = "expected"
|
|
16
|
+
METADATA = "metadata"
|
|
17
|
+
IGNORE = "ignore"
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
class OutputType(StrEnum):
|
|
21
|
+
"""The kind of output the prompted model is expected to produce."""
|
|
22
|
+
|
|
23
|
+
LABEL = "label"
|
|
24
|
+
JSON = "json"
|
|
25
|
+
TEXT = "text"
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
class ApprovalMode(StrEnum):
|
|
29
|
+
AUTO = "auto"
|
|
30
|
+
MANUAL = "manual"
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
class ModelRole(StrEnum):
|
|
34
|
+
"""Independent model roles; each can use a different provider and model."""
|
|
35
|
+
|
|
36
|
+
TARGET = "target"
|
|
37
|
+
OPTIMIZER = "optimizer"
|
|
38
|
+
ANALYZER = "analyzer"
|
|
39
|
+
JUDGE = "judge"
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
class RuleType(StrEnum):
|
|
43
|
+
"""Deterministic checks applied to each model output."""
|
|
44
|
+
|
|
45
|
+
MAX_CHARS = "max_chars"
|
|
46
|
+
MIN_CHARS = "min_chars"
|
|
47
|
+
MAX_WORDS = "max_words"
|
|
48
|
+
MIN_WORDS = "min_words"
|
|
49
|
+
MATCHES = "matches"
|
|
50
|
+
NOT_MATCHES = "not_matches"
|
|
51
|
+
CONTAINS_ALL = "contains_all"
|
|
52
|
+
CONTAINS_NONE = "contains_none"
|
strop/config/io.py
ADDED
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
"""Reading and writing strop.yaml."""
|
|
2
|
+
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
|
|
5
|
+
from pydantic import ValidationError
|
|
6
|
+
from ruamel.yaml import YAML
|
|
7
|
+
from ruamel.yaml.comments import CommentedMap
|
|
8
|
+
from ruamel.yaml.error import YAMLError
|
|
9
|
+
|
|
10
|
+
from strop.config.root import StropConfig
|
|
11
|
+
from strop.errors import ConfigError
|
|
12
|
+
|
|
13
|
+
HEADER = "Strop project configuration. Command-line flags override these values."
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def load_config(path: Path) -> StropConfig:
|
|
17
|
+
"""Read and validate ``path``; raise ``ConfigError`` with readable messages."""
|
|
18
|
+
try:
|
|
19
|
+
text = path.read_text(encoding="utf-8")
|
|
20
|
+
except FileNotFoundError:
|
|
21
|
+
raise ConfigError(f"config file not found: {path} (run 'strop init' first)") from None
|
|
22
|
+
try:
|
|
23
|
+
data = YAML(typ="safe").load(text)
|
|
24
|
+
except YAMLError as exc:
|
|
25
|
+
raise ConfigError(f"{path} is not valid YAML: {exc}") from exc
|
|
26
|
+
if not isinstance(data, dict):
|
|
27
|
+
raise ConfigError(f"{path} must contain a mapping at the top level")
|
|
28
|
+
try:
|
|
29
|
+
return StropConfig.model_validate(data)
|
|
30
|
+
except ValidationError as exc:
|
|
31
|
+
raise ConfigError(f"invalid configuration in {path}:\n{format_errors(exc)}") from exc
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def dump_config(config: StropConfig, path: Path, *, force: bool = False) -> None:
|
|
35
|
+
"""Write ``config`` to ``path``; refuse to overwrite unless ``force`` is set."""
|
|
36
|
+
if path.exists() and not force:
|
|
37
|
+
raise ConfigError(f"{path} already exists; use --force to overwrite it")
|
|
38
|
+
document = CommentedMap(config.model_dump(mode="json", exclude_none=True))
|
|
39
|
+
document.yaml_set_start_comment(HEADER)
|
|
40
|
+
yaml = YAML()
|
|
41
|
+
yaml.default_flow_style = False
|
|
42
|
+
with path.open("w", encoding="utf-8") as handle:
|
|
43
|
+
yaml.dump(document, handle)
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def format_errors(exc: ValidationError) -> str:
|
|
47
|
+
"""One indented ``field.path: message`` line per validation error."""
|
|
48
|
+
lines = []
|
|
49
|
+
for error in exc.errors():
|
|
50
|
+
location = ".".join(str(part) for part in error["loc"]) or "(root)"
|
|
51
|
+
lines.append(f" {location}: {error['msg']}")
|
|
52
|
+
return "\n".join(lines)
|
strop/config/limits.py
ADDED
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
"""Budgets and execution settings."""
|
|
2
|
+
|
|
3
|
+
from pydantic import Field
|
|
4
|
+
|
|
5
|
+
from strop.config.base import StrictModel
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class BudgetsConfig(StrictModel):
|
|
9
|
+
"""Upper limits for a run. ``None`` disables a limit."""
|
|
10
|
+
|
|
11
|
+
max_total_model_calls: int | None = Field(default=500, gt=0)
|
|
12
|
+
max_total_cost_usd: float | None = Field(default=None, gt=0)
|
|
13
|
+
max_wall_time_minutes: float | None = Field(default=60.0, gt=0)
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
class ExecutionConfig(StrictModel):
|
|
17
|
+
batch_size: int = Field(default=10, ge=1)
|
|
18
|
+
max_concurrency: int = Field(default=3, ge=1)
|
|
19
|
+
retry_attempts: int = Field(default=3, ge=1)
|
strop/config/llm.py
ADDED
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
"""Model selection. Strop never defaults to a specific provider or model."""
|
|
2
|
+
|
|
3
|
+
from typing import Any
|
|
4
|
+
|
|
5
|
+
from pydantic import Field
|
|
6
|
+
|
|
7
|
+
from strop.config.base import StrictModel
|
|
8
|
+
from strop.config.enums import ModelRole
|
|
9
|
+
from strop.errors import ConfigError
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
class ModelSpec(StrictModel):
|
|
13
|
+
"""One model, given as a LiteLLM model string such as ``<provider>/<model>``."""
|
|
14
|
+
|
|
15
|
+
model: str = Field(min_length=1)
|
|
16
|
+
temperature: float = Field(default=0.0, ge=0.0, le=2.0)
|
|
17
|
+
max_tokens: int | None = Field(default=None, gt=0)
|
|
18
|
+
timeout_s: float = Field(default=60.0, gt=0)
|
|
19
|
+
params: dict[str, Any] = Field(default_factory=dict)
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class ModelsConfig(StrictModel):
|
|
23
|
+
target: ModelSpec
|
|
24
|
+
optimizer: ModelSpec | None = None
|
|
25
|
+
analyzer: ModelSpec | None = None
|
|
26
|
+
judge: ModelSpec | None = None
|
|
27
|
+
|
|
28
|
+
def resolve(self, role: ModelRole) -> ModelSpec:
|
|
29
|
+
"""Return the model for ``role``; the analyzer falls back to the optimizer."""
|
|
30
|
+
specs = {
|
|
31
|
+
ModelRole.TARGET: self.target,
|
|
32
|
+
ModelRole.OPTIMIZER: self.optimizer,
|
|
33
|
+
ModelRole.ANALYZER: self.analyzer or self.optimizer,
|
|
34
|
+
ModelRole.JUDGE: self.judge,
|
|
35
|
+
}
|
|
36
|
+
spec = specs[role]
|
|
37
|
+
if spec is None:
|
|
38
|
+
raise ConfigError(
|
|
39
|
+
f"no model configured for the '{role.value}' role; set models.{role.value}.model "
|
|
40
|
+
f"in strop.yaml (strop init --{role.value}-model writes it)"
|
|
41
|
+
)
|
|
42
|
+
return spec
|
strop/config/metrics.py
ADDED
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
"""Success definition: primary metric, rule checks, and constraints a candidate must keep."""
|
|
2
|
+
|
|
3
|
+
import re
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
from typing import Literal, Self
|
|
6
|
+
|
|
7
|
+
from pydantic import Field, model_validator
|
|
8
|
+
|
|
9
|
+
from strop.config.base import StrictModel
|
|
10
|
+
from strop.config.enums import RuleType
|
|
11
|
+
|
|
12
|
+
_COUNT_RULES = {RuleType.MAX_CHARS, RuleType.MIN_CHARS, RuleType.MAX_WORDS, RuleType.MIN_WORDS}
|
|
13
|
+
_PATTERN_RULES = {RuleType.MATCHES, RuleType.NOT_MATCHES}
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
class Bound(StrictModel):
|
|
17
|
+
"""An inclusive lower and/or upper limit on a metric value."""
|
|
18
|
+
|
|
19
|
+
min: float | None = None
|
|
20
|
+
max: float | None = None
|
|
21
|
+
|
|
22
|
+
@model_validator(mode="after")
|
|
23
|
+
def _check_bounds(self) -> Self:
|
|
24
|
+
if self.min is None and self.max is None:
|
|
25
|
+
raise ValueError("a constraint needs 'min' or 'max'")
|
|
26
|
+
if self.min is not None and self.max is not None and self.min > self.max:
|
|
27
|
+
raise ValueError("'min' must not be greater than 'max'")
|
|
28
|
+
return self
|
|
29
|
+
|
|
30
|
+
def allows(self, value: float) -> bool:
|
|
31
|
+
return (self.min is None or value >= self.min) and (self.max is None or value <= self.max)
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
class RuleSpec(StrictModel):
|
|
35
|
+
"""One deterministic output check. The rule's pass rate becomes the metric ``rule:<name>``."""
|
|
36
|
+
|
|
37
|
+
type: RuleType
|
|
38
|
+
value: int | str | list[str]
|
|
39
|
+
|
|
40
|
+
@model_validator(mode="after")
|
|
41
|
+
def _check_value(self) -> Self:
|
|
42
|
+
if self.type in _COUNT_RULES:
|
|
43
|
+
if not isinstance(self.value, int) or isinstance(self.value, bool) or self.value < 0:
|
|
44
|
+
raise ValueError(f"rule type '{self.type}' needs a non-negative integer value")
|
|
45
|
+
elif self.type in _PATTERN_RULES:
|
|
46
|
+
if not isinstance(self.value, str):
|
|
47
|
+
raise ValueError(f"rule type '{self.type}' needs a regular expression string")
|
|
48
|
+
try:
|
|
49
|
+
re.compile(self.value)
|
|
50
|
+
except re.error as exc:
|
|
51
|
+
raise ValueError(f"invalid regular expression: {exc}") from exc
|
|
52
|
+
elif not isinstance(self.value, list) or not self.value:
|
|
53
|
+
raise ValueError(f"rule type '{self.type}' needs a non-empty list of strings")
|
|
54
|
+
return self
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
class PythonMetricConfig(StrictModel):
|
|
58
|
+
"""A user-written metric: a Python file defining ``score(output, row)``."""
|
|
59
|
+
|
|
60
|
+
file: Path
|
|
61
|
+
timeout_s: float = Field(default=120.0, gt=0)
|
|
62
|
+
direction: Literal["maximize", "minimize"] = "maximize"
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
class JudgeSpec(StrictModel):
|
|
66
|
+
"""A rubric scored by a judge model. The score is normalized to the range 0 to 1."""
|
|
67
|
+
|
|
68
|
+
criteria: str = Field(min_length=1)
|
|
69
|
+
scale_min: int = 1
|
|
70
|
+
scale_max: int = 5
|
|
71
|
+
instructions: str = ""
|
|
72
|
+
|
|
73
|
+
@model_validator(mode="after")
|
|
74
|
+
def _check_scale(self) -> Self:
|
|
75
|
+
if self.scale_max <= self.scale_min:
|
|
76
|
+
raise ValueError("scale_max must be greater than scale_min")
|
|
77
|
+
return self
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
class MetricsConfig(StrictModel):
|
|
81
|
+
primary: str = Field(min_length=1)
|
|
82
|
+
secondary: tuple[str, ...] = ()
|
|
83
|
+
constraints: dict[str, Bound] = Field(default_factory=dict)
|
|
84
|
+
rules: dict[str, RuleSpec] = Field(default_factory=dict)
|
|
85
|
+
python: PythonMetricConfig | None = None
|
|
86
|
+
judges: dict[str, JudgeSpec] = Field(default_factory=dict)
|