strop-cli 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (137) hide show
  1. strop/__init__.py +5 -0
  2. strop/__main__.py +5 -0
  3. strop/analysis/__init__.py +17 -0
  4. strop/analysis/classify.py +52 -0
  5. strop/analysis/evidence.py +52 -0
  6. strop/analysis/facts.py +74 -0
  7. strop/analysis/patterns.py +90 -0
  8. strop/cli/__init__.py +5 -0
  9. strop/cli/analyze.py +93 -0
  10. strop/cli/app.py +82 -0
  11. strop/cli/data.py +111 -0
  12. strop/cli/evaluate.py +145 -0
  13. strop/cli/export.py +74 -0
  14. strop/cli/final.py +106 -0
  15. strop/cli/init.py +131 -0
  16. strop/cli/optimize.py +156 -0
  17. strop/cli/options.py +20 -0
  18. strop/cli/progress.py +18 -0
  19. strop/cli/prompts.py +110 -0
  20. strop/cli/status.py +68 -0
  21. strop/cli/versions.py +123 -0
  22. strop/config/__init__.py +16 -0
  23. strop/config/base.py +9 -0
  24. strop/config/constraints.py +33 -0
  25. strop/config/dataset.py +63 -0
  26. strop/config/enums.py +52 -0
  27. strop/config/io.py +52 -0
  28. strop/config/limits.py +19 -0
  29. strop/config/llm.py +42 -0
  30. strop/config/metrics.py +86 -0
  31. strop/config/optimization.py +23 -0
  32. strop/config/output.py +31 -0
  33. strop/config/overrides.py +22 -0
  34. strop/config/root.py +36 -0
  35. strop/config/rules.py +67 -0
  36. strop/config/split.py +23 -0
  37. strop/data/__init__.py +58 -0
  38. strop/data/loaders.py +85 -0
  39. strop/data/models.py +44 -0
  40. strop/data/rows.py +99 -0
  41. strop/data/splitting.py +103 -0
  42. strop/env.py +44 -0
  43. strop/errors.py +59 -0
  44. strop/evaluation/__init__.py +1 -0
  45. strop/evaluation/budget.py +67 -0
  46. strop/evaluation/budgeted.py +23 -0
  47. strop/evaluation/examples.py +49 -0
  48. strop/evaluation/models.py +105 -0
  49. strop/evaluation/parsing.py +96 -0
  50. strop/evaluation/runner.py +107 -0
  51. strop/formatting.py +14 -0
  52. strop/llm/__init__.py +7 -0
  53. strop/llm/caching.py +28 -0
  54. strop/llm/client.py +15 -0
  55. strop/llm/factory.py +14 -0
  56. strop/llm/litellm_client.py +90 -0
  57. strop/llm/mock.py +43 -0
  58. strop/llm/retry.py +44 -0
  59. strop/llm/structured.py +52 -0
  60. strop/llm/types.py +59 -0
  61. strop/log.py +49 -0
  62. strop/metrics/__init__.py +1 -0
  63. strop/metrics/basic.py +30 -0
  64. strop/metrics/classification.py +164 -0
  65. strop/metrics/judge.py +130 -0
  66. strop/metrics/python_metric.py +101 -0
  67. strop/metrics/python_runner.py +65 -0
  68. strop/metrics/registry.py +115 -0
  69. strop/metrics/rules.py +55 -0
  70. strop/metrics/structured.py +109 -0
  71. strop/metrics/suite.py +109 -0
  72. strop/metrics/text.py +55 -0
  73. strop/metrics/types.py +106 -0
  74. strop/optimization/__init__.py +25 -0
  75. strop/optimization/acceptance.py +170 -0
  76. strop/optimization/generator.py +92 -0
  77. strop/optimization/meta_prompt.py +101 -0
  78. strop/optimization/types.py +69 -0
  79. strop/prompts/__init__.py +18 -0
  80. strop/prompts/diff.py +139 -0
  81. strop/prompts/files.py +92 -0
  82. strop/prompts/models.py +51 -0
  83. strop/prompts/render.py +16 -0
  84. strop/prompts/template.py +45 -0
  85. strop/prompts/validation.py +31 -0
  86. strop/prompts/versions.py +34 -0
  87. strop/py.typed +0 -0
  88. strop/reporting/__init__.py +6 -0
  89. strop/reporting/console.py +20 -0
  90. strop/reporting/diff_view.py +102 -0
  91. strop/reporting/evaluation_view.py +141 -0
  92. strop/reporting/final_view.py +109 -0
  93. strop/reporting/prompt_export.py +58 -0
  94. strop/reporting/report_markdown.py +157 -0
  95. strop/reporting/run_view.py +118 -0
  96. strop/services/__init__.py +1 -0
  97. strop/services/analysis.py +124 -0
  98. strop/services/clean.py +38 -0
  99. strop/services/compare.py +60 -0
  100. strop/services/context.py +70 -0
  101. strop/services/data.py +65 -0
  102. strop/services/evaluate.py +105 -0
  103. strop/services/evaluator.py +169 -0
  104. strop/services/export.py +90 -0
  105. strop/services/final_test.py +69 -0
  106. strop/services/metrics.py +89 -0
  107. strop/services/optimize.py +187 -0
  108. strop/services/partitions.py +46 -0
  109. strop/services/project.py +55 -0
  110. strop/services/prompts.py +76 -0
  111. strop/services/report.py +194 -0
  112. strop/services/review.py +43 -0
  113. strop/services/scaffold.py +79 -0
  114. strop/services/turn.py +234 -0
  115. strop/storage/__init__.py +6 -0
  116. strop/storage/cache.py +45 -0
  117. strop/storage/database.py +45 -0
  118. strop/storage/datasets.py +129 -0
  119. strop/storage/evaluations.py +160 -0
  120. strop/storage/migrations/001_initial.sql +5 -0
  121. strop/storage/migrations/002_datasets.sql +27 -0
  122. strop/storage/migrations/003_prompt_versions.sql +22 -0
  123. strop/storage/migrations/004_llm_cache.sql +6 -0
  124. strop/storage/migrations/005_evaluations.sql +37 -0
  125. strop/storage/migrations/006_runs.sql +52 -0
  126. strop/storage/migrations/007_example_feedback.sql +2 -0
  127. strop/storage/migrations/__init__.py +5 -0
  128. strop/storage/migrator.py +91 -0
  129. strop/storage/paths.py +30 -0
  130. strop/storage/prompts.py +96 -0
  131. strop/storage/run_models.py +75 -0
  132. strop/storage/runs.py +263 -0
  133. strop_cli-0.1.0.dist-info/METADATA +242 -0
  134. strop_cli-0.1.0.dist-info/RECORD +137 -0
  135. strop_cli-0.1.0.dist-info/WHEEL +4 -0
  136. strop_cli-0.1.0.dist-info/entry_points.txt +2 -0
  137. strop_cli-0.1.0.dist-info/licenses/LICENSE +21 -0
strop/cli/prompts.py ADDED
@@ -0,0 +1,110 @@
1
+ """The ``strop prompt`` command group."""
2
+
3
+ from pathlib import Path
4
+ from typing import Annotated
5
+
6
+ import typer
7
+
8
+ from strop.cli.options import DirOption, JsonOption, emit_json
9
+ from strop.prompts.versions import PromptVersion
10
+ from strop.services.project import Project, open_project
11
+ from strop.services.prompts import add_version_from_file, load_project_prompt, resolve_version
12
+ from strop.storage.prompts import PromptRepository
13
+
14
+ prompt_app = typer.Typer(help="Add and inspect prompt versions.", no_args_is_help=True)
15
+
16
+
17
+ @prompt_app.command("add")
18
+ def add_command(
19
+ file: Annotated[Path, typer.Option(help="Prompt file holding the edited prompt.")],
20
+ parent: Annotated[
21
+ str | None, typer.Option(help="Version it derives from; default: best.")
22
+ ] = None,
23
+ reason: Annotated[str, typer.Option(help="Why this edit was made.")] = "",
24
+ use: Annotated[bool, typer.Option("--use", help="Make this version the best prompt.")] = False,
25
+ directory: DirOption = Path(),
26
+ ) -> None:
27
+ """Save an edited prompt file as a new immutable version."""
28
+ project = open_project(directory)
29
+ with project.database() as conn:
30
+ repo = PromptRepository(conn)
31
+ version = add_version_from_file(project, repo, file, parent=parent, reason=reason, use=use)
32
+ typer.echo(f"Saved {version.ref} (parent v{version.parent_number})")
33
+ if use:
34
+ typer.echo(f"{version.ref} is now the best prompt.")
35
+
36
+
37
+ @prompt_app.command("show")
38
+ def show_command(
39
+ version: Annotated[
40
+ str | None, typer.Argument(help="Version such as v2; default: best.")
41
+ ] = None,
42
+ as_json: JsonOption = False,
43
+ directory: DirOption = Path(),
44
+ ) -> None:
45
+ """Print the messages of a prompt version."""
46
+ project = open_project(directory)
47
+ with project.database() as conn:
48
+ repo = PromptRepository(conn)
49
+ if version is None and repo.first() is None:
50
+ _show_unregistered(project, as_json)
51
+ return
52
+ selected = resolve_version(repo, version)
53
+ is_best = repo.best_number() == selected.number
54
+ if as_json:
55
+ emit_json(_payload(selected, is_best=is_best))
56
+ return
57
+ typer.echo(_header(selected, is_best=is_best))
58
+ for message in selected.template.messages:
59
+ editable = "editable" if selected.template.is_editable(message) else "frozen"
60
+ typer.echo(f"--- {message.role} ({editable})")
61
+ typer.echo(message.content.rstrip("\n"))
62
+
63
+
64
+ def _show_unregistered(project: Project, as_json: bool) -> None:
65
+ """Before the first run there are no versions yet, so show the prompt file itself."""
66
+ template = load_project_prompt(project)
67
+ file = project.config.prompt.file
68
+ if as_json:
69
+ emit_json(
70
+ {
71
+ "version": None,
72
+ "file": str(file),
73
+ "messages": [
74
+ {"role": m.role, "content": m.content, "editable": template.is_editable(m)}
75
+ for m in template.messages
76
+ ],
77
+ }
78
+ )
79
+ return
80
+ typer.echo(f"{file} (not registered yet; it becomes v1 on the first run)")
81
+ for message in template.messages:
82
+ editable = "editable" if template.is_editable(message) else "frozen"
83
+ typer.echo(f"--- {message.role} ({editable})")
84
+ typer.echo(message.content.rstrip("\n"))
85
+
86
+
87
+ def _header(version: PromptVersion, *, is_best: bool) -> str:
88
+ parent = f"parent v{version.parent_number}" if version.parent_number else "no parent"
89
+ best = ", best" if is_best else ""
90
+ summary = f"\nSummary: {version.change_summary}" if version.change_summary else ""
91
+ return f"{version.ref} ({parent}, {version.source}{best}){summary}"
92
+
93
+
94
+ def _payload(version: PromptVersion, *, is_best: bool) -> dict[str, object]:
95
+ return {
96
+ "version": version.number,
97
+ "parent": version.parent_number,
98
+ "source": version.source,
99
+ "best": is_best,
100
+ "change_summary": version.change_summary,
101
+ "reason": version.reason,
102
+ "messages": [
103
+ {
104
+ "role": m.role,
105
+ "content": m.content,
106
+ "editable": version.template.is_editable(m),
107
+ }
108
+ for m in version.template.messages
109
+ ],
110
+ }
strop/cli/status.py ADDED
@@ -0,0 +1,68 @@
1
+ """The ``strop status`` command."""
2
+
3
+ from pathlib import Path
4
+
5
+ import typer
6
+
7
+ from strop.cli.options import DirOption, JsonOption, emit_json
8
+ from strop.formatting import fmt
9
+ from strop.services.project import open_project
10
+ from strop.storage.datasets import DatasetRepository
11
+ from strop.storage.evaluations import EvaluationRepository
12
+ from strop.storage.prompts import PromptRepository
13
+
14
+ RECENT = 5
15
+
16
+
17
+ def status_command(as_json: JsonOption = False, directory: DirOption = Path()) -> None:
18
+ """Show the dataset split, the best prompt, and the latest evaluations."""
19
+ project = open_project(directory)
20
+ with project.database() as conn:
21
+ datasets = DatasetRepository(conn)
22
+ split = datasets.latest_split()
23
+ counts = datasets.partition_counts(split.id) if split else {}
24
+ prompts = PromptRepository(conn)
25
+ versions = prompts.list()
26
+ best = prompts.best_number()
27
+ recent = EvaluationRepository(conn).find(limit=RECENT)
28
+ payload = {
29
+ "project": project.config.project.name,
30
+ "split": (
31
+ {"id": split.id, "seed": split.seed, "counts": {p.value: n for p, n in counts.items()}}
32
+ if split
33
+ else None
34
+ ),
35
+ "prompt_versions": len(versions),
36
+ "best_version": best,
37
+ "recent_evaluations": [
38
+ {
39
+ "id": e.id,
40
+ "version": e.prompt_number,
41
+ "partition": e.partition.value,
42
+ "kind": e.kind,
43
+ "primary_metric": e.report.primary,
44
+ "value": e.report.primary_value.value,
45
+ "samples": e.sample_count,
46
+ }
47
+ for e in recent
48
+ ],
49
+ }
50
+ if as_json:
51
+ emit_json(payload)
52
+ return
53
+ typer.echo(f"Project: {project.config.project.name}")
54
+ if split:
55
+ sizes = ", ".join(f"{p.value} {n}" for p, n in counts.items())
56
+ typer.echo(f"Split {split.id} (seed {split.seed}): {sizes}")
57
+ else:
58
+ typer.echo("Split: none yet (run 'strop data split')")
59
+ if versions:
60
+ typer.echo(f"Prompt: v{best} is the best of {len(versions)} version(s)")
61
+ else:
62
+ typer.echo("Prompt: no versions yet (registered on the first run)")
63
+ for evaluation in recent:
64
+ typer.echo(
65
+ f" #{evaluation.id} v{evaluation.prompt_number} {evaluation.partition.value} "
66
+ f"{evaluation.kind}: {evaluation.report.primary}="
67
+ f"{fmt(evaluation.report.primary_value.value)} ({evaluation.sample_count} examples)"
68
+ )
strop/cli/versions.py ADDED
@@ -0,0 +1,123 @@
1
+ """Top-level commands about prompt history: ``diff``, ``history``, ``rollback``."""
2
+
3
+ from pathlib import Path
4
+ from typing import Annotated
5
+
6
+ import typer
7
+
8
+ from strop.cli.options import DirOption, JsonOption, emit_json
9
+ from strop.errors import ConfigError
10
+ from strop.prompts.diff import diff_templates
11
+ from strop.prompts.versions import PromptVersion
12
+ from strop.reporting import ColorMode, make_console, render_diff
13
+ from strop.services.project import open_project
14
+ from strop.services.prompts import resolve_version
15
+ from strop.storage.prompts import PromptRepository
16
+ from strop.storage.runs import CandidateRecord, RunRepository
17
+
18
+ SUMMARY_WIDTH = 60
19
+
20
+
21
+ def diff_command(
22
+ old: Annotated[str | None, typer.Argument(help="Older version, such as v1.")] = None,
23
+ new: Annotated[str | None, typer.Argument(help="Newer version; default: best.")] = None,
24
+ color: Annotated[ColorMode, typer.Option(help="When to use color.")] = ColorMode.AUTO,
25
+ as_json: JsonOption = False,
26
+ directory: DirOption = Path(),
27
+ ) -> None:
28
+ """Show a colored diff of two prompt versions.
29
+
30
+ With no arguments: the first prompt (v1) against the best prompt. With one argument:
31
+ that version against its parent.
32
+ """
33
+ project = open_project(directory)
34
+ with project.database() as conn:
35
+ repo = PromptRepository(conn)
36
+ before, after = _pick_pair(repo, old, new)
37
+ diff = diff_templates(before.template, after.template)
38
+ if as_json:
39
+ emit_json({"from": before.ref, "to": after.ref, **diff.to_payload()})
40
+ return
41
+ console = make_console(color)
42
+ console.print(f"{before.ref} -> {after.ref}", soft_wrap=True, highlight=False)
43
+ render_diff(diff, console)
44
+
45
+
46
+ def history_command(
47
+ as_json: JsonOption = False,
48
+ directory: DirOption = Path(),
49
+ ) -> None:
50
+ """List every prompt version with the decision made about it. The best is marked with *."""
51
+ project = open_project(directory)
52
+ with project.database() as conn:
53
+ repo = PromptRepository(conn)
54
+ runs = RunRepository(conn)
55
+ versions = repo.list()
56
+ best = repo.best_number()
57
+ decisions = {v.number: runs.for_version(v.number) for v in versions}
58
+ if as_json:
59
+ emit_json({"best": best, "versions": [_row(v, decisions[v.number]) for v in versions]})
60
+ return
61
+ if not versions:
62
+ typer.echo("No prompt versions yet. The prompt file is registered as v1 on the first run.")
63
+ return
64
+ for version in versions:
65
+ marker = "*" if version.number == best else " "
66
+ parent = f"v{version.parent_number}" if version.parent_number else "-"
67
+ decision = decisions[version.number]
68
+ status = decision.status if decision else "-"
69
+ delta = (
70
+ f"{decision.primary_delta:+.4f}"
71
+ if decision and decision.primary_delta is not None
72
+ else "-"
73
+ )
74
+ summary = version.change_summary[:SUMMARY_WIDTH]
75
+ typer.echo(
76
+ f"{marker} {version.ref:<5} {version.source:<10} parent {parent:<5} "
77
+ f"{status:<9} {delta:<8} {summary}"
78
+ )
79
+
80
+
81
+ def rollback_command(
82
+ version: Annotated[str, typer.Argument(help="Version to make the best prompt.")],
83
+ directory: DirOption = Path(),
84
+ ) -> None:
85
+ """Make an earlier version the best prompt. No version is deleted or changed."""
86
+ project = open_project(directory)
87
+ with project.database() as conn:
88
+ repo = PromptRepository(conn)
89
+ target = resolve_version(repo, version)
90
+ previous = repo.best_number()
91
+ repo.set_best(target.number)
92
+ was = f" (was v{previous})" if previous else ""
93
+ typer.echo(f"Best prompt is now {target.ref}{was}. No versions were deleted.")
94
+
95
+
96
+ def _pick_pair(
97
+ repo: PromptRepository, old: str | None, new: str | None
98
+ ) -> tuple[PromptVersion, PromptVersion]:
99
+ if old is None:
100
+ first = repo.first()
101
+ if first is None:
102
+ raise ConfigError("no prompt versions yet; run 'strop baseline' first")
103
+ return first, resolve_version(repo, None)
104
+ if new is not None:
105
+ return resolve_version(repo, old), resolve_version(repo, new)
106
+ selected = resolve_version(repo, old)
107
+ if selected.parent_number is None:
108
+ raise ConfigError(f"{selected.ref} has no parent; name two versions to compare")
109
+ return repo.get(selected.parent_number), selected
110
+
111
+
112
+ def _row(version: PromptVersion, decision: CandidateRecord | None) -> dict[str, object]:
113
+ return {
114
+ "status": decision.status if decision else None,
115
+ "primary_delta": decision.primary_delta if decision else None,
116
+ "decision_reason": decision.reason if decision else None,
117
+ "version": version.number,
118
+ "parent": version.parent_number,
119
+ "source": version.source,
120
+ "change_summary": version.change_summary,
121
+ "reason": version.reason,
122
+ "created_at": version.created_at,
123
+ }
@@ -0,0 +1,16 @@
1
+ """Typed project configuration and its YAML loader."""
2
+
3
+ from strop.config.enums import ApprovalMode, ColumnRole, DatasetFormat, ModelRole, OutputType
4
+ from strop.config.io import dump_config, load_config
5
+ from strop.config.root import StropConfig
6
+
7
+ __all__ = [
8
+ "ApprovalMode",
9
+ "ColumnRole",
10
+ "DatasetFormat",
11
+ "ModelRole",
12
+ "OutputType",
13
+ "StropConfig",
14
+ "dump_config",
15
+ "load_config",
16
+ ]
strop/config/base.py ADDED
@@ -0,0 +1,9 @@
1
+ """Shared base class for configuration models."""
2
+
3
+ from pydantic import BaseModel, ConfigDict
4
+
5
+
6
+ class StrictModel(BaseModel):
7
+ """Immutable model that rejects unknown keys, so typos in strop.yaml fail loudly."""
8
+
9
+ model_config = ConfigDict(extra="forbid", frozen=True)
@@ -0,0 +1,33 @@
1
+ """Parsing of ``--constraint`` flags such as ``valid_output_rate>=0.98``."""
2
+
3
+ import re
4
+ from collections.abc import Iterable
5
+
6
+ from strop.config.metrics import Bound
7
+ from strop.errors import ConfigError
8
+
9
+ _EXPRESSION = re.compile(r"^\s*(?P<metric>[\w:.\-]+)\s*(?P<op>>=|<=)\s*(?P<value>[-+0-9.eE]+)\s*$")
10
+
11
+
12
+ def parse_constraints(expressions: Iterable[str]) -> dict[str, Bound]:
13
+ """Turn ``metric>=x`` and ``metric<=y`` expressions into per-metric bounds.
14
+
15
+ Two expressions for the same metric (one ``>=``, one ``<=``) are merged.
16
+ """
17
+ limits: dict[str, dict[str, float]] = {}
18
+ for text in expressions:
19
+ match = _EXPRESSION.match(text)
20
+ if match is None:
21
+ raise ConfigError(
22
+ f"invalid constraint '{text}'; use the form 'metric>=0.9' or 'metric<=0.1'"
23
+ )
24
+ try:
25
+ value = float(match["value"])
26
+ except ValueError:
27
+ raise ConfigError(f"invalid number in constraint '{text}'") from None
28
+ side = "min" if match["op"] == ">=" else "max"
29
+ bounds = limits.setdefault(match["metric"], {})
30
+ if side in bounds:
31
+ raise ConfigError(f"constraint on '{match['metric']}' sets '{side}' more than once")
32
+ bounds[side] = value
33
+ return {metric: Bound(**bounds) for metric, bounds in limits.items()}
@@ -0,0 +1,63 @@
1
+ """Dataset location and column roles."""
2
+
3
+ from pathlib import Path
4
+ from typing import Self
5
+
6
+ from pydantic import model_validator
7
+
8
+ from strop.config.base import StrictModel
9
+ from strop.config.enums import ColumnRole, DatasetFormat
10
+ from strop.errors import ConfigError
11
+
12
+ _SUFFIX_FORMATS = {
13
+ ".csv": DatasetFormat.CSV,
14
+ ".jsonl": DatasetFormat.JSONL,
15
+ ".ndjson": DatasetFormat.JSONL,
16
+ }
17
+
18
+
19
+ class DatasetConfig(StrictModel):
20
+ path: Path
21
+ format: DatasetFormat | None = None
22
+ id_column: str | None = None
23
+ columns: dict[str, ColumnRole]
24
+
25
+ @model_validator(mode="after")
26
+ def _check_roles(self) -> Self:
27
+ roles = list(self.columns.values())
28
+ if ColumnRole.INPUT not in roles:
29
+ raise ValueError("at least one column must have the role 'input'")
30
+ if roles.count(ColumnRole.EXPECTED) > 1:
31
+ raise ValueError("at most one column may have the role 'expected'")
32
+ for name in self._columns_with(ColumnRole.INPUT):
33
+ if not name.isidentifier():
34
+ raise ValueError(
35
+ f"input column '{name}' cannot be used as a prompt variable; "
36
+ "rename it to letters, digits, and underscores, not starting with a digit"
37
+ )
38
+ return self
39
+
40
+ @property
41
+ def resolved_format(self) -> DatasetFormat:
42
+ """The explicit format, or the one implied by the file extension."""
43
+ if self.format is not None:
44
+ return self.format
45
+ try:
46
+ return _SUFFIX_FORMATS[self.path.suffix.lower()]
47
+ except KeyError:
48
+ raise ConfigError(
49
+ f"cannot infer the dataset format from '{self.path.name}'; "
50
+ "set dataset.format to csv or jsonl"
51
+ ) from None
52
+
53
+ @property
54
+ def input_columns(self) -> tuple[str, ...]:
55
+ return self._columns_with(ColumnRole.INPUT)
56
+
57
+ @property
58
+ def expected_column(self) -> str | None:
59
+ expected = self._columns_with(ColumnRole.EXPECTED)
60
+ return expected[0] if expected else None
61
+
62
+ def _columns_with(self, role: ColumnRole) -> tuple[str, ...]:
63
+ return tuple(name for name, value in self.columns.items() if value == role)
strop/config/enums.py ADDED
@@ -0,0 +1,52 @@
1
+ """Enumerations used across the configuration sections."""
2
+
3
+ from enum import StrEnum
4
+
5
+
6
+ class DatasetFormat(StrEnum):
7
+ CSV = "csv"
8
+ JSONL = "jsonl"
9
+
10
+
11
+ class ColumnRole(StrEnum):
12
+ """What a dataset column is used for."""
13
+
14
+ INPUT = "input"
15
+ EXPECTED = "expected"
16
+ METADATA = "metadata"
17
+ IGNORE = "ignore"
18
+
19
+
20
+ class OutputType(StrEnum):
21
+ """The kind of output the prompted model is expected to produce."""
22
+
23
+ LABEL = "label"
24
+ JSON = "json"
25
+ TEXT = "text"
26
+
27
+
28
+ class ApprovalMode(StrEnum):
29
+ AUTO = "auto"
30
+ MANUAL = "manual"
31
+
32
+
33
+ class ModelRole(StrEnum):
34
+ """Independent model roles; each can use a different provider and model."""
35
+
36
+ TARGET = "target"
37
+ OPTIMIZER = "optimizer"
38
+ ANALYZER = "analyzer"
39
+ JUDGE = "judge"
40
+
41
+
42
+ class RuleType(StrEnum):
43
+ """Deterministic checks applied to each model output."""
44
+
45
+ MAX_CHARS = "max_chars"
46
+ MIN_CHARS = "min_chars"
47
+ MAX_WORDS = "max_words"
48
+ MIN_WORDS = "min_words"
49
+ MATCHES = "matches"
50
+ NOT_MATCHES = "not_matches"
51
+ CONTAINS_ALL = "contains_all"
52
+ CONTAINS_NONE = "contains_none"
strop/config/io.py ADDED
@@ -0,0 +1,52 @@
1
+ """Reading and writing strop.yaml."""
2
+
3
+ from pathlib import Path
4
+
5
+ from pydantic import ValidationError
6
+ from ruamel.yaml import YAML
7
+ from ruamel.yaml.comments import CommentedMap
8
+ from ruamel.yaml.error import YAMLError
9
+
10
+ from strop.config.root import StropConfig
11
+ from strop.errors import ConfigError
12
+
13
+ HEADER = "Strop project configuration. Command-line flags override these values."
14
+
15
+
16
+ def load_config(path: Path) -> StropConfig:
17
+ """Read and validate ``path``; raise ``ConfigError`` with readable messages."""
18
+ try:
19
+ text = path.read_text(encoding="utf-8")
20
+ except FileNotFoundError:
21
+ raise ConfigError(f"config file not found: {path} (run 'strop init' first)") from None
22
+ try:
23
+ data = YAML(typ="safe").load(text)
24
+ except YAMLError as exc:
25
+ raise ConfigError(f"{path} is not valid YAML: {exc}") from exc
26
+ if not isinstance(data, dict):
27
+ raise ConfigError(f"{path} must contain a mapping at the top level")
28
+ try:
29
+ return StropConfig.model_validate(data)
30
+ except ValidationError as exc:
31
+ raise ConfigError(f"invalid configuration in {path}:\n{format_errors(exc)}") from exc
32
+
33
+
34
+ def dump_config(config: StropConfig, path: Path, *, force: bool = False) -> None:
35
+ """Write ``config`` to ``path``; refuse to overwrite unless ``force`` is set."""
36
+ if path.exists() and not force:
37
+ raise ConfigError(f"{path} already exists; use --force to overwrite it")
38
+ document = CommentedMap(config.model_dump(mode="json", exclude_none=True))
39
+ document.yaml_set_start_comment(HEADER)
40
+ yaml = YAML()
41
+ yaml.default_flow_style = False
42
+ with path.open("w", encoding="utf-8") as handle:
43
+ yaml.dump(document, handle)
44
+
45
+
46
+ def format_errors(exc: ValidationError) -> str:
47
+ """One indented ``field.path: message`` line per validation error."""
48
+ lines = []
49
+ for error in exc.errors():
50
+ location = ".".join(str(part) for part in error["loc"]) or "(root)"
51
+ lines.append(f" {location}: {error['msg']}")
52
+ return "\n".join(lines)
strop/config/limits.py ADDED
@@ -0,0 +1,19 @@
1
+ """Budgets and execution settings."""
2
+
3
+ from pydantic import Field
4
+
5
+ from strop.config.base import StrictModel
6
+
7
+
8
+ class BudgetsConfig(StrictModel):
9
+ """Upper limits for a run. ``None`` disables a limit."""
10
+
11
+ max_total_model_calls: int | None = Field(default=500, gt=0)
12
+ max_total_cost_usd: float | None = Field(default=None, gt=0)
13
+ max_wall_time_minutes: float | None = Field(default=60.0, gt=0)
14
+
15
+
16
+ class ExecutionConfig(StrictModel):
17
+ batch_size: int = Field(default=10, ge=1)
18
+ max_concurrency: int = Field(default=3, ge=1)
19
+ retry_attempts: int = Field(default=3, ge=1)
strop/config/llm.py ADDED
@@ -0,0 +1,42 @@
1
+ """Model selection. Strop never defaults to a specific provider or model."""
2
+
3
+ from typing import Any
4
+
5
+ from pydantic import Field
6
+
7
+ from strop.config.base import StrictModel
8
+ from strop.config.enums import ModelRole
9
+ from strop.errors import ConfigError
10
+
11
+
12
+ class ModelSpec(StrictModel):
13
+ """One model, given as a LiteLLM model string such as ``<provider>/<model>``."""
14
+
15
+ model: str = Field(min_length=1)
16
+ temperature: float = Field(default=0.0, ge=0.0, le=2.0)
17
+ max_tokens: int | None = Field(default=None, gt=0)
18
+ timeout_s: float = Field(default=60.0, gt=0)
19
+ params: dict[str, Any] = Field(default_factory=dict)
20
+
21
+
22
+ class ModelsConfig(StrictModel):
23
+ target: ModelSpec
24
+ optimizer: ModelSpec | None = None
25
+ analyzer: ModelSpec | None = None
26
+ judge: ModelSpec | None = None
27
+
28
+ def resolve(self, role: ModelRole) -> ModelSpec:
29
+ """Return the model for ``role``; the analyzer falls back to the optimizer."""
30
+ specs = {
31
+ ModelRole.TARGET: self.target,
32
+ ModelRole.OPTIMIZER: self.optimizer,
33
+ ModelRole.ANALYZER: self.analyzer or self.optimizer,
34
+ ModelRole.JUDGE: self.judge,
35
+ }
36
+ spec = specs[role]
37
+ if spec is None:
38
+ raise ConfigError(
39
+ f"no model configured for the '{role.value}' role; set models.{role.value}.model "
40
+ f"in strop.yaml (strop init --{role.value}-model writes it)"
41
+ )
42
+ return spec
@@ -0,0 +1,86 @@
1
+ """Success definition: primary metric, rule checks, and constraints a candidate must keep."""
2
+
3
+ import re
4
+ from pathlib import Path
5
+ from typing import Literal, Self
6
+
7
+ from pydantic import Field, model_validator
8
+
9
+ from strop.config.base import StrictModel
10
+ from strop.config.enums import RuleType
11
+
12
+ _COUNT_RULES = {RuleType.MAX_CHARS, RuleType.MIN_CHARS, RuleType.MAX_WORDS, RuleType.MIN_WORDS}
13
+ _PATTERN_RULES = {RuleType.MATCHES, RuleType.NOT_MATCHES}
14
+
15
+
16
+ class Bound(StrictModel):
17
+ """An inclusive lower and/or upper limit on a metric value."""
18
+
19
+ min: float | None = None
20
+ max: float | None = None
21
+
22
+ @model_validator(mode="after")
23
+ def _check_bounds(self) -> Self:
24
+ if self.min is None and self.max is None:
25
+ raise ValueError("a constraint needs 'min' or 'max'")
26
+ if self.min is not None and self.max is not None and self.min > self.max:
27
+ raise ValueError("'min' must not be greater than 'max'")
28
+ return self
29
+
30
+ def allows(self, value: float) -> bool:
31
+ return (self.min is None or value >= self.min) and (self.max is None or value <= self.max)
32
+
33
+
34
+ class RuleSpec(StrictModel):
35
+ """One deterministic output check. The rule's pass rate becomes the metric ``rule:<name>``."""
36
+
37
+ type: RuleType
38
+ value: int | str | list[str]
39
+
40
+ @model_validator(mode="after")
41
+ def _check_value(self) -> Self:
42
+ if self.type in _COUNT_RULES:
43
+ if not isinstance(self.value, int) or isinstance(self.value, bool) or self.value < 0:
44
+ raise ValueError(f"rule type '{self.type}' needs a non-negative integer value")
45
+ elif self.type in _PATTERN_RULES:
46
+ if not isinstance(self.value, str):
47
+ raise ValueError(f"rule type '{self.type}' needs a regular expression string")
48
+ try:
49
+ re.compile(self.value)
50
+ except re.error as exc:
51
+ raise ValueError(f"invalid regular expression: {exc}") from exc
52
+ elif not isinstance(self.value, list) or not self.value:
53
+ raise ValueError(f"rule type '{self.type}' needs a non-empty list of strings")
54
+ return self
55
+
56
+
57
+ class PythonMetricConfig(StrictModel):
58
+ """A user-written metric: a Python file defining ``score(output, row)``."""
59
+
60
+ file: Path
61
+ timeout_s: float = Field(default=120.0, gt=0)
62
+ direction: Literal["maximize", "minimize"] = "maximize"
63
+
64
+
65
+ class JudgeSpec(StrictModel):
66
+ """A rubric scored by a judge model. The score is normalized to the range 0 to 1."""
67
+
68
+ criteria: str = Field(min_length=1)
69
+ scale_min: int = 1
70
+ scale_max: int = 5
71
+ instructions: str = ""
72
+
73
+ @model_validator(mode="after")
74
+ def _check_scale(self) -> Self:
75
+ if self.scale_max <= self.scale_min:
76
+ raise ValueError("scale_max must be greater than scale_min")
77
+ return self
78
+
79
+
80
+ class MetricsConfig(StrictModel):
81
+ primary: str = Field(min_length=1)
82
+ secondary: tuple[str, ...] = ()
83
+ constraints: dict[str, Bound] = Field(default_factory=dict)
84
+ rules: dict[str, RuleSpec] = Field(default_factory=dict)
85
+ python: PythonMetricConfig | None = None
86
+ judges: dict[str, JudgeSpec] = Field(default_factory=dict)