strop-cli 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- strop/__init__.py +5 -0
- strop/__main__.py +5 -0
- strop/analysis/__init__.py +17 -0
- strop/analysis/classify.py +52 -0
- strop/analysis/evidence.py +52 -0
- strop/analysis/facts.py +74 -0
- strop/analysis/patterns.py +90 -0
- strop/cli/__init__.py +5 -0
- strop/cli/analyze.py +93 -0
- strop/cli/app.py +82 -0
- strop/cli/data.py +111 -0
- strop/cli/evaluate.py +145 -0
- strop/cli/export.py +74 -0
- strop/cli/final.py +106 -0
- strop/cli/init.py +131 -0
- strop/cli/optimize.py +156 -0
- strop/cli/options.py +20 -0
- strop/cli/progress.py +18 -0
- strop/cli/prompts.py +110 -0
- strop/cli/status.py +68 -0
- strop/cli/versions.py +123 -0
- strop/config/__init__.py +16 -0
- strop/config/base.py +9 -0
- strop/config/constraints.py +33 -0
- strop/config/dataset.py +63 -0
- strop/config/enums.py +52 -0
- strop/config/io.py +52 -0
- strop/config/limits.py +19 -0
- strop/config/llm.py +42 -0
- strop/config/metrics.py +86 -0
- strop/config/optimization.py +23 -0
- strop/config/output.py +31 -0
- strop/config/overrides.py +22 -0
- strop/config/root.py +36 -0
- strop/config/rules.py +67 -0
- strop/config/split.py +23 -0
- strop/data/__init__.py +58 -0
- strop/data/loaders.py +85 -0
- strop/data/models.py +44 -0
- strop/data/rows.py +99 -0
- strop/data/splitting.py +103 -0
- strop/env.py +44 -0
- strop/errors.py +59 -0
- strop/evaluation/__init__.py +1 -0
- strop/evaluation/budget.py +67 -0
- strop/evaluation/budgeted.py +23 -0
- strop/evaluation/examples.py +49 -0
- strop/evaluation/models.py +105 -0
- strop/evaluation/parsing.py +96 -0
- strop/evaluation/runner.py +107 -0
- strop/formatting.py +14 -0
- strop/llm/__init__.py +7 -0
- strop/llm/caching.py +28 -0
- strop/llm/client.py +15 -0
- strop/llm/factory.py +14 -0
- strop/llm/litellm_client.py +90 -0
- strop/llm/mock.py +43 -0
- strop/llm/retry.py +44 -0
- strop/llm/structured.py +52 -0
- strop/llm/types.py +59 -0
- strop/log.py +49 -0
- strop/metrics/__init__.py +1 -0
- strop/metrics/basic.py +30 -0
- strop/metrics/classification.py +164 -0
- strop/metrics/judge.py +130 -0
- strop/metrics/python_metric.py +101 -0
- strop/metrics/python_runner.py +65 -0
- strop/metrics/registry.py +115 -0
- strop/metrics/rules.py +55 -0
- strop/metrics/structured.py +109 -0
- strop/metrics/suite.py +109 -0
- strop/metrics/text.py +55 -0
- strop/metrics/types.py +106 -0
- strop/optimization/__init__.py +25 -0
- strop/optimization/acceptance.py +170 -0
- strop/optimization/generator.py +92 -0
- strop/optimization/meta_prompt.py +101 -0
- strop/optimization/types.py +69 -0
- strop/prompts/__init__.py +18 -0
- strop/prompts/diff.py +139 -0
- strop/prompts/files.py +92 -0
- strop/prompts/models.py +51 -0
- strop/prompts/render.py +16 -0
- strop/prompts/template.py +45 -0
- strop/prompts/validation.py +31 -0
- strop/prompts/versions.py +34 -0
- strop/py.typed +0 -0
- strop/reporting/__init__.py +6 -0
- strop/reporting/console.py +20 -0
- strop/reporting/diff_view.py +102 -0
- strop/reporting/evaluation_view.py +141 -0
- strop/reporting/final_view.py +109 -0
- strop/reporting/prompt_export.py +58 -0
- strop/reporting/report_markdown.py +157 -0
- strop/reporting/run_view.py +118 -0
- strop/services/__init__.py +1 -0
- strop/services/analysis.py +124 -0
- strop/services/clean.py +38 -0
- strop/services/compare.py +60 -0
- strop/services/context.py +70 -0
- strop/services/data.py +65 -0
- strop/services/evaluate.py +105 -0
- strop/services/evaluator.py +169 -0
- strop/services/export.py +90 -0
- strop/services/final_test.py +69 -0
- strop/services/metrics.py +89 -0
- strop/services/optimize.py +187 -0
- strop/services/partitions.py +46 -0
- strop/services/project.py +55 -0
- strop/services/prompts.py +76 -0
- strop/services/report.py +194 -0
- strop/services/review.py +43 -0
- strop/services/scaffold.py +79 -0
- strop/services/turn.py +234 -0
- strop/storage/__init__.py +6 -0
- strop/storage/cache.py +45 -0
- strop/storage/database.py +45 -0
- strop/storage/datasets.py +129 -0
- strop/storage/evaluations.py +160 -0
- strop/storage/migrations/001_initial.sql +5 -0
- strop/storage/migrations/002_datasets.sql +27 -0
- strop/storage/migrations/003_prompt_versions.sql +22 -0
- strop/storage/migrations/004_llm_cache.sql +6 -0
- strop/storage/migrations/005_evaluations.sql +37 -0
- strop/storage/migrations/006_runs.sql +52 -0
- strop/storage/migrations/007_example_feedback.sql +2 -0
- strop/storage/migrations/__init__.py +5 -0
- strop/storage/migrator.py +91 -0
- strop/storage/paths.py +30 -0
- strop/storage/prompts.py +96 -0
- strop/storage/run_models.py +75 -0
- strop/storage/runs.py +263 -0
- strop_cli-0.1.0.dist-info/METADATA +242 -0
- strop_cli-0.1.0.dist-info/RECORD +137 -0
- strop_cli-0.1.0.dist-info/WHEEL +4 -0
- strop_cli-0.1.0.dist-info/entry_points.txt +2 -0
- strop_cli-0.1.0.dist-info/licenses/LICENSE +21 -0
strop/prompts/models.py
ADDED
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
"""Prompt template types."""
|
|
2
|
+
|
|
3
|
+
import hashlib
|
|
4
|
+
import json
|
|
5
|
+
from dataclasses import dataclass
|
|
6
|
+
|
|
7
|
+
from strop.llm.types import Role
|
|
8
|
+
from strop.prompts.template import find_variables
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
@dataclass(frozen=True, slots=True)
|
|
12
|
+
class PromptMessage:
|
|
13
|
+
role: Role
|
|
14
|
+
content: str
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
@dataclass(frozen=True, slots=True)
|
|
18
|
+
class PromptTemplate:
|
|
19
|
+
"""Chat messages plus the roles whose messages the optimizer may rewrite."""
|
|
20
|
+
|
|
21
|
+
messages: tuple[PromptMessage, ...]
|
|
22
|
+
optimize: frozenset[Role]
|
|
23
|
+
|
|
24
|
+
@property
|
|
25
|
+
def variables(self) -> tuple[str, ...]:
|
|
26
|
+
"""Variable names used anywhere in the template, in order of first use."""
|
|
27
|
+
names: dict[str, None] = {}
|
|
28
|
+
for message in self.messages:
|
|
29
|
+
names.update(dict.fromkeys(find_variables(message.content)))
|
|
30
|
+
return tuple(names)
|
|
31
|
+
|
|
32
|
+
def is_editable(self, message: PromptMessage) -> bool:
|
|
33
|
+
return message.role in self.optimize
|
|
34
|
+
|
|
35
|
+
def to_json(self) -> str:
|
|
36
|
+
"""Canonical JSON used for storage and hashing."""
|
|
37
|
+
payload = {
|
|
38
|
+
"messages": [{"role": m.role, "content": m.content} for m in self.messages],
|
|
39
|
+
"optimize": sorted(self.optimize),
|
|
40
|
+
}
|
|
41
|
+
return json.dumps(payload, sort_keys=True, ensure_ascii=False)
|
|
42
|
+
|
|
43
|
+
@classmethod
|
|
44
|
+
def from_json(cls, text: str) -> "PromptTemplate":
|
|
45
|
+
data = json.loads(text)
|
|
46
|
+
messages = tuple(PromptMessage(m["role"], m["content"]) for m in data["messages"])
|
|
47
|
+
return cls(messages, frozenset(data["optimize"]))
|
|
48
|
+
|
|
49
|
+
@property
|
|
50
|
+
def content_hash(self) -> str:
|
|
51
|
+
return hashlib.sha256(self.to_json().encode("utf-8")).hexdigest()
|
strop/prompts/render.py
ADDED
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
"""Rendering a template for one dataset row."""
|
|
2
|
+
|
|
3
|
+
from collections.abc import Mapping
|
|
4
|
+
|
|
5
|
+
from strop.errors import ConfigError
|
|
6
|
+
from strop.llm.types import Message
|
|
7
|
+
from strop.prompts.models import PromptTemplate
|
|
8
|
+
from strop.prompts.template import substitute
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def render_messages(template: PromptTemplate, values: Mapping[str, str]) -> list[Message]:
|
|
12
|
+
"""Fill every placeholder from ``values`` and return chat messages for the model."""
|
|
13
|
+
missing = [name for name in template.variables if name not in values]
|
|
14
|
+
if missing:
|
|
15
|
+
raise ConfigError(f"no value for prompt variable(s): {', '.join(missing)}")
|
|
16
|
+
return [Message(role=m.role, content=substitute(m.content, values)) for m in template.messages]
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
"""Variable discovery and substitution for ``{{name}}`` placeholders."""
|
|
2
|
+
|
|
3
|
+
import re
|
|
4
|
+
from collections.abc import Mapping
|
|
5
|
+
|
|
6
|
+
from strop.errors import ConfigError
|
|
7
|
+
|
|
8
|
+
_VARIABLE = re.compile(r"\{\{\s*([A-Za-z_]\w*)\s*\}\}")
|
|
9
|
+
_FIXED = re.compile(r"<strop:fixed>(.*?)</strop:fixed>", re.DOTALL)
|
|
10
|
+
FIXED_OPEN = "<strop:fixed>"
|
|
11
|
+
FIXED_CLOSE = "</strop:fixed>"
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def find_variables(text: str) -> list[str]:
|
|
15
|
+
"""Distinct variable names in ``text``, in order of first use."""
|
|
16
|
+
return list(dict.fromkeys(_VARIABLE.findall(text)))
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def fixed_spans(text: str) -> list[str]:
|
|
20
|
+
"""Contents of every ``<strop:fixed>`` span, in order."""
|
|
21
|
+
return _FIXED.findall(text)
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def strip_markers(text: str) -> str:
|
|
25
|
+
"""Remove fixed-span markers, keeping their contents."""
|
|
26
|
+
return _FIXED.sub(lambda match: match.group(1), text)
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def has_balanced_markers(text: str) -> bool:
|
|
30
|
+
return text.count(FIXED_OPEN) == text.count(FIXED_CLOSE) == len(_FIXED.findall(text))
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def substitute(text: str, values: Mapping[str, str]) -> str:
|
|
34
|
+
"""Replace placeholders in one pass, so substituted values are never expanded again."""
|
|
35
|
+
return _VARIABLE.sub(lambda match: values[match.group(1)], strip_markers(text))
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def check_variables(used: tuple[str, ...], available: tuple[str, ...]) -> None:
|
|
39
|
+
"""Raise if the prompt uses a variable that no dataset input column provides."""
|
|
40
|
+
unknown = [name for name in used if name not in available]
|
|
41
|
+
if unknown:
|
|
42
|
+
raise ConfigError(
|
|
43
|
+
f"the prompt uses variables with no matching input column: {', '.join(unknown)}; "
|
|
44
|
+
f"input columns: {', '.join(available)}"
|
|
45
|
+
)
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
"""Checks that an optimizer-proposed revision only changed what it was allowed to change."""
|
|
2
|
+
|
|
3
|
+
from collections import Counter
|
|
4
|
+
|
|
5
|
+
from strop.prompts.models import PromptTemplate
|
|
6
|
+
from strop.prompts.template import fixed_spans
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def validate_revision(parent: PromptTemplate, candidate: PromptTemplate) -> list[str]:
|
|
10
|
+
"""Return a list of rule violations; an empty list means the revision is acceptable."""
|
|
11
|
+
if [m.role for m in parent.messages] != [m.role for m in candidate.messages]:
|
|
12
|
+
return ["the message structure (roles and order) must not change"]
|
|
13
|
+
problems = []
|
|
14
|
+
if parent.optimize != candidate.optimize:
|
|
15
|
+
problems.append("the set of editable roles must not change")
|
|
16
|
+
for index, (old, new) in enumerate(zip(parent.messages, candidate.messages, strict=True), 1):
|
|
17
|
+
label = f"message {index} ({old.role})"
|
|
18
|
+
if not parent.is_editable(old):
|
|
19
|
+
if old.content != new.content:
|
|
20
|
+
problems.append(f"{label} is frozen but was modified")
|
|
21
|
+
continue
|
|
22
|
+
if not new.content.strip():
|
|
23
|
+
problems.append(f"{label} is empty")
|
|
24
|
+
if Counter(fixed_spans(old.content)) != Counter(fixed_spans(new.content)):
|
|
25
|
+
problems.append(f"{label} changed or removed a <strop:fixed> span")
|
|
26
|
+
if set(parent.variables) != set(candidate.variables):
|
|
27
|
+
problems.append(
|
|
28
|
+
f"template variables changed: expected {sorted(parent.variables)}, "
|
|
29
|
+
f"got {sorted(candidate.variables)}"
|
|
30
|
+
)
|
|
31
|
+
return problems
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
"""Prompt version records."""
|
|
2
|
+
|
|
3
|
+
from dataclasses import dataclass
|
|
4
|
+
from typing import Literal
|
|
5
|
+
|
|
6
|
+
from strop.errors import ConfigError
|
|
7
|
+
from strop.prompts.models import PromptTemplate
|
|
8
|
+
|
|
9
|
+
Source = Literal["user", "optimizer"]
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
@dataclass(frozen=True, slots=True)
|
|
13
|
+
class PromptVersion:
|
|
14
|
+
"""One immutable revision of the prompt, identified as ``v<number>``."""
|
|
15
|
+
|
|
16
|
+
number: int
|
|
17
|
+
parent_number: int | None
|
|
18
|
+
template: PromptTemplate
|
|
19
|
+
change_summary: str
|
|
20
|
+
reason: str
|
|
21
|
+
source: Source
|
|
22
|
+
created_at: str
|
|
23
|
+
|
|
24
|
+
@property
|
|
25
|
+
def ref(self) -> str:
|
|
26
|
+
return f"v{self.number}"
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def parse_version_ref(text: str) -> int:
|
|
30
|
+
"""Parse ``v3`` or ``3`` into a version number."""
|
|
31
|
+
digits = text.strip().lower().removeprefix("v")
|
|
32
|
+
if not digits.isdigit() or int(digits) < 1:
|
|
33
|
+
raise ConfigError(f"invalid prompt version '{text}'; use a form like v3")
|
|
34
|
+
return int(digits)
|
strop/py.typed
ADDED
|
File without changes
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
"""Console construction with an explicit color mode."""
|
|
2
|
+
|
|
3
|
+
from enum import StrEnum
|
|
4
|
+
|
|
5
|
+
from rich.console import Console
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class ColorMode(StrEnum):
|
|
9
|
+
AUTO = "auto"
|
|
10
|
+
ALWAYS = "always"
|
|
11
|
+
NEVER = "never"
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def make_console(mode: ColorMode = ColorMode.AUTO) -> Console:
|
|
15
|
+
"""A stdout console. ``auto`` colors only on a terminal and respects ``NO_COLOR``."""
|
|
16
|
+
if mode is ColorMode.ALWAYS:
|
|
17
|
+
return Console(force_terminal=True, color_system="standard", highlight=False)
|
|
18
|
+
if mode is ColorMode.NEVER:
|
|
19
|
+
return Console(force_terminal=False, no_color=True, color_system=None, highlight=False)
|
|
20
|
+
return Console(highlight=False)
|
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
"""Colored terminal rendering of a prompt diff.
|
|
2
|
+
|
|
3
|
+
Without color (pipes, files, ``NO_COLOR``) the output is a plain unified diff.
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
import difflib
|
|
7
|
+
import re
|
|
8
|
+
|
|
9
|
+
from rich.console import Console
|
|
10
|
+
from rich.text import Text
|
|
11
|
+
|
|
12
|
+
from strop.prompts.diff import DiffLine, MessageDiff, PromptDiff
|
|
13
|
+
|
|
14
|
+
_TOKEN = re.compile(r"\s+|\w+|[^\w\s]")
|
|
15
|
+
_STYLES = {
|
|
16
|
+
"remove": ("red", "bold red reverse"),
|
|
17
|
+
"add": ("green", "bold green reverse"),
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def render_diff(diff: PromptDiff, console: Console) -> None:
|
|
22
|
+
"""Print ``diff`` with colored lines, word-level highlights, and a summary line."""
|
|
23
|
+
for message in diff.messages:
|
|
24
|
+
console.print(_heading(message), soft_wrap=True)
|
|
25
|
+
for text in _message_lines(message.lines):
|
|
26
|
+
console.print(text, soft_wrap=True)
|
|
27
|
+
console.print(Text(_summary(diff), style="dim"), soft_wrap=True)
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def _heading(message: MessageDiff) -> Text:
|
|
31
|
+
style = "bold" if message.status in {"changed", "added", "removed"} else "dim"
|
|
32
|
+
return Text(f"== message {message.index} ({message.role}): {message.status}", style=style)
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def _message_lines(lines: tuple[DiffLine, ...]) -> list[Text]:
|
|
36
|
+
out: list[Text] = []
|
|
37
|
+
index = 0
|
|
38
|
+
while index < len(lines):
|
|
39
|
+
removed, added, index = _take_change_block(lines, index)
|
|
40
|
+
if removed or added:
|
|
41
|
+
out.extend(_change_block(removed, added))
|
|
42
|
+
continue
|
|
43
|
+
line = lines[index]
|
|
44
|
+
out.append(_plain_line(line))
|
|
45
|
+
index += 1
|
|
46
|
+
return out
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def _take_change_block(lines: tuple[DiffLine, ...], start: int) -> tuple[list[str], list[str], int]:
|
|
50
|
+
index = start
|
|
51
|
+
removed: list[str] = []
|
|
52
|
+
added: list[str] = []
|
|
53
|
+
while index < len(lines) and lines[index].kind == "remove":
|
|
54
|
+
removed.append(lines[index].text)
|
|
55
|
+
index += 1
|
|
56
|
+
while index < len(lines) and lines[index].kind == "add":
|
|
57
|
+
added.append(lines[index].text)
|
|
58
|
+
index += 1
|
|
59
|
+
return removed, added, index
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def _change_block(removed: list[str], added: list[str]) -> list[Text]:
|
|
63
|
+
old_texts = [Text("-")] * len(removed)
|
|
64
|
+
new_texts = [Text("+")] * len(added)
|
|
65
|
+
for position in range(max(len(removed), len(added))):
|
|
66
|
+
before = removed[position] if position < len(removed) else None
|
|
67
|
+
after = added[position] if position < len(added) else None
|
|
68
|
+
if before is not None and after is not None:
|
|
69
|
+
old_texts[position], new_texts[position] = _word_diff(before, after)
|
|
70
|
+
elif before is not None:
|
|
71
|
+
old_texts[position] = Text(f"-{before}", style=_STYLES["remove"][0])
|
|
72
|
+
elif after is not None:
|
|
73
|
+
new_texts[position] = Text(f"+{after}", style=_STYLES["add"][0])
|
|
74
|
+
return [*old_texts, *new_texts]
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def _word_diff(before: str, after: str) -> tuple[Text, Text]:
|
|
78
|
+
old_tokens = _TOKEN.findall(before)
|
|
79
|
+
new_tokens = _TOKEN.findall(after)
|
|
80
|
+
old_text = Text("-", style=_STYLES["remove"][0])
|
|
81
|
+
new_text = Text("+", style=_STYLES["add"][0])
|
|
82
|
+
matcher = difflib.SequenceMatcher(None, old_tokens, new_tokens, autojunk=False)
|
|
83
|
+
for tag, i1, i2, j1, j2 in matcher.get_opcodes():
|
|
84
|
+
changed = tag != "equal"
|
|
85
|
+
old_text.append("".join(old_tokens[i1:i2]), _STYLES["remove"][1] if changed else None)
|
|
86
|
+
new_text.append("".join(new_tokens[j1:j2]), _STYLES["add"][1] if changed else None)
|
|
87
|
+
return old_text, new_text
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def _plain_line(line: DiffLine) -> Text:
|
|
91
|
+
if line.kind == "hunk":
|
|
92
|
+
return Text(line.text, style="cyan")
|
|
93
|
+
return Text(f"{line.prefix}{line.text}", style="dim")
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def _summary(diff: PromptDiff) -> str:
|
|
97
|
+
stats = diff.stats
|
|
98
|
+
change = stats.approx_token_change
|
|
99
|
+
return (
|
|
100
|
+
f"Summary: +{stats.added_lines} line(s), -{stats.removed_lines} line(s), "
|
|
101
|
+
f"prompt {stats.chars_before} -> {stats.chars_after} chars (~{change:+d} tokens)"
|
|
102
|
+
)
|
|
@@ -0,0 +1,141 @@
|
|
|
1
|
+
"""Terminal rendering and JSON payloads for evaluations."""
|
|
2
|
+
|
|
3
|
+
from collections import Counter
|
|
4
|
+
from collections.abc import Sequence
|
|
5
|
+
from typing import Any
|
|
6
|
+
|
|
7
|
+
from rich.console import Console
|
|
8
|
+
from rich.table import Table
|
|
9
|
+
from rich.text import Text
|
|
10
|
+
|
|
11
|
+
from strop.evaluation.models import EvaluationRecord, ExampleOutcome, UsageSummary
|
|
12
|
+
from strop.formatting import fmt, shorten
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def render_evaluation(
|
|
16
|
+
console: Console,
|
|
17
|
+
record: EvaluationRecord,
|
|
18
|
+
outcomes: Sequence[ExampleOutcome],
|
|
19
|
+
*,
|
|
20
|
+
version_ref: str,
|
|
21
|
+
show_errors: int = 5,
|
|
22
|
+
) -> None:
|
|
23
|
+
"""Print the metrics, error summary, per-class scores, usage, and sample failures."""
|
|
24
|
+
title = (
|
|
25
|
+
f"Evaluation {record.id}: {version_ref} on {record.partition.value} "
|
|
26
|
+
f"({record.sample_count} examples, {record.kind})"
|
|
27
|
+
)
|
|
28
|
+
console.print(Text(title, style="bold"), soft_wrap=True)
|
|
29
|
+
console.print(_metrics_table(record))
|
|
30
|
+
console.print(_error_summary(outcomes), soft_wrap=True)
|
|
31
|
+
classes = _per_class_table(record)
|
|
32
|
+
if classes is not None:
|
|
33
|
+
console.print(classes)
|
|
34
|
+
console.print(Text(usage_line(record.usage), style="dim"), soft_wrap=True)
|
|
35
|
+
for line in failure_lines(outcomes, show_errors):
|
|
36
|
+
console.print(line, soft_wrap=True, markup=False)
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def _metrics_table(record: EvaluationRecord) -> Table:
|
|
40
|
+
table = Table(box=None, pad_edge=False, header_style="bold")
|
|
41
|
+
for column in ("Metric", "Value", "Samples", "Note"):
|
|
42
|
+
table.add_column(column)
|
|
43
|
+
for name, metric in record.report.values.items():
|
|
44
|
+
label = f"{name} (primary)" if name == record.report.primary else name
|
|
45
|
+
note = "subjective, LLM-judged" if metric.subjective else ""
|
|
46
|
+
table.add_row(label, fmt(metric.value), str(metric.samples), note)
|
|
47
|
+
return table
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def _error_summary(outcomes: Sequence[ExampleOutcome]) -> str:
|
|
51
|
+
codes = Counter(o.error for o in outcomes if o.error)
|
|
52
|
+
wrong = sum(1 for o in outcomes if _is_wrong(o))
|
|
53
|
+
parts = [f"{code} {count}" for code, count in sorted(codes.items())]
|
|
54
|
+
if wrong:
|
|
55
|
+
parts.append(f"valid but wrong {wrong}")
|
|
56
|
+
return f"Output problems: {', '.join(parts)}" if parts else "No output problems."
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def _is_wrong(outcome: ExampleOutcome) -> bool:
|
|
60
|
+
return (
|
|
61
|
+
outcome.value is not None
|
|
62
|
+
and outcome.expected is not None
|
|
63
|
+
and outcome.value != outcome.expected
|
|
64
|
+
)
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def _per_class_table(record: EvaluationRecord) -> Table | None:
|
|
68
|
+
per_class = record.report.primary_value.details.get("per_class")
|
|
69
|
+
if not per_class:
|
|
70
|
+
return None
|
|
71
|
+
table = Table(
|
|
72
|
+
box=None, pad_edge=False, header_style="bold", title="Per class", title_justify="left"
|
|
73
|
+
)
|
|
74
|
+
for column in ("Class", "Precision", "Recall", "F1", "Support"):
|
|
75
|
+
table.add_column(column)
|
|
76
|
+
for label, scores in per_class.items():
|
|
77
|
+
table.add_row(
|
|
78
|
+
label,
|
|
79
|
+
fmt(scores["precision"]),
|
|
80
|
+
fmt(scores["recall"]),
|
|
81
|
+
fmt(scores["f1"]),
|
|
82
|
+
str(scores["support"]),
|
|
83
|
+
)
|
|
84
|
+
return table
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def usage_line(usage: UsageSummary) -> str:
|
|
88
|
+
cost = "unavailable" if usage.cost_usd is None else f"${usage.cost_usd:.4f}"
|
|
89
|
+
if usage.cost_usd is not None and not usage.cost_complete:
|
|
90
|
+
cost += " (partial)"
|
|
91
|
+
tokens = "unavailable" if usage.total_tokens is None else f"{usage.total_tokens:,}"
|
|
92
|
+
latency = "unavailable" if usage.mean_latency_s is None else f"{usage.mean_latency_s:.2f}s"
|
|
93
|
+
return (
|
|
94
|
+
f"Calls: {usage.calls} ({usage.cached_calls} cached) | Tokens: {tokens} | "
|
|
95
|
+
f"Cost: {cost} | Mean latency: {latency}"
|
|
96
|
+
)
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def failure_lines(outcomes: Sequence[ExampleOutcome], limit: int) -> list[str]:
|
|
100
|
+
"""One line per failing example, up to ``limit``, with a trailer for the rest."""
|
|
101
|
+
failing = [o for o in outcomes if o.error or _is_wrong(o)]
|
|
102
|
+
if limit <= 0 or not failing:
|
|
103
|
+
return []
|
|
104
|
+
lines = ["Examples that failed:"]
|
|
105
|
+
for outcome in failing[:limit]:
|
|
106
|
+
if outcome.error:
|
|
107
|
+
got = shorten(outcome.detail or outcome.raw_output)
|
|
108
|
+
lines.append(f" {outcome.row_id}: {outcome.error} ({got})")
|
|
109
|
+
else:
|
|
110
|
+
lines.append(
|
|
111
|
+
f" {outcome.row_id}: expected {shorten(outcome.expected or '')!r}, "
|
|
112
|
+
f"got {shorten(outcome.value or '')!r}"
|
|
113
|
+
)
|
|
114
|
+
if len(failing) > limit:
|
|
115
|
+
lines.append(f" ... and {len(failing) - limit} more")
|
|
116
|
+
return lines
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
def record_payload(record: EvaluationRecord, version_ref: str) -> dict[str, Any]:
|
|
120
|
+
usage = record.usage
|
|
121
|
+
return {
|
|
122
|
+
"evaluation": record.id,
|
|
123
|
+
"version": version_ref,
|
|
124
|
+
"partition": record.partition.value,
|
|
125
|
+
"kind": record.kind,
|
|
126
|
+
"samples": record.sample_count,
|
|
127
|
+
"errors": record.error_count,
|
|
128
|
+
"primary_metric": record.report.primary,
|
|
129
|
+
"metrics": {
|
|
130
|
+
name: {"value": m.value, "samples": m.samples, "subjective": m.subjective}
|
|
131
|
+
for name, m in record.report.values.items()
|
|
132
|
+
},
|
|
133
|
+
"usage": {
|
|
134
|
+
"calls": usage.calls,
|
|
135
|
+
"cached_calls": usage.cached_calls,
|
|
136
|
+
"total_tokens": usage.total_tokens,
|
|
137
|
+
"cost_usd": usage.cost_usd,
|
|
138
|
+
"cost_complete": usage.cost_complete,
|
|
139
|
+
"mean_latency_s": usage.mean_latency_s,
|
|
140
|
+
},
|
|
141
|
+
}
|
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
"""Terminal rendering and JSON payloads for the test and compare commands."""
|
|
2
|
+
|
|
3
|
+
import math
|
|
4
|
+
from typing import Any
|
|
5
|
+
|
|
6
|
+
from rich.console import Console
|
|
7
|
+
from rich.table import Table
|
|
8
|
+
from rich.text import Text
|
|
9
|
+
|
|
10
|
+
from strop.evaluation.models import EvaluationRecord
|
|
11
|
+
from strop.formatting import fmt
|
|
12
|
+
from strop.reporting.evaluation_view import record_payload, usage_line
|
|
13
|
+
from strop.services.compare import Comparison
|
|
14
|
+
from strop.services.final_test import FinalTest
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def render_final_test(console: Console, result: FinalTest) -> None:
|
|
18
|
+
"""Summarize the held-out result next to the validation result it should be read against."""
|
|
19
|
+
record = result.record
|
|
20
|
+
primary = record.report.primary
|
|
21
|
+
console.print(Text("Held-out test result", style="bold"), soft_wrap=True)
|
|
22
|
+
table = Table(box=None, pad_edge=False, header_style="bold")
|
|
23
|
+
for column in ("Prompt", "Partition", primary, "Examples"):
|
|
24
|
+
table.add_column(column)
|
|
25
|
+
if result.baseline is not None:
|
|
26
|
+
initial, base = result.baseline
|
|
27
|
+
table.add_row(initial.ref + " (initial)", "test", _value(base), str(base.sample_count))
|
|
28
|
+
if result.validation is not None:
|
|
29
|
+
table.add_row(
|
|
30
|
+
result.version.ref,
|
|
31
|
+
"validation",
|
|
32
|
+
_value(result.validation),
|
|
33
|
+
str(result.validation.sample_count),
|
|
34
|
+
)
|
|
35
|
+
table.add_row(result.version.ref, "test", _value(record), str(record.sample_count))
|
|
36
|
+
console.print(table)
|
|
37
|
+
console.print(Text(usage_line(record.usage), style="dim"), soft_wrap=True)
|
|
38
|
+
note = standard_error_note(record)
|
|
39
|
+
if note:
|
|
40
|
+
console.print(note, soft_wrap=True, highlight=False)
|
|
41
|
+
console.print(
|
|
42
|
+
"Do not change the prompt based on this result: the test set is now used up.",
|
|
43
|
+
soft_wrap=True,
|
|
44
|
+
highlight=False,
|
|
45
|
+
)
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def standard_error_note(record: EvaluationRecord) -> str | None:
|
|
49
|
+
"""A rough noise estimate for rate-like metrics, so small test sets are not over-read."""
|
|
50
|
+
value = record.report.primary_value.value
|
|
51
|
+
n = record.report.primary_value.samples
|
|
52
|
+
if value is None or n == 0 or not 0.0 <= value <= 1.0:
|
|
53
|
+
return None
|
|
54
|
+
error = math.sqrt(value * (1 - value) / n)
|
|
55
|
+
return (
|
|
56
|
+
f"Note: with {n} test examples, differences smaller than about {error:.2f} "
|
|
57
|
+
f"(one standard error for a rate near {value:.2f}) may be noise."
|
|
58
|
+
)
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def render_comparison(console: Console, comparison: Comparison) -> None:
|
|
62
|
+
title = f"{comparison.version.ref} on {comparison.partition.value}, same rows for every model"
|
|
63
|
+
console.print(Text(title, style="bold"), soft_wrap=True)
|
|
64
|
+
table = Table(box=None, pad_edge=False, header_style="bold")
|
|
65
|
+
primary = comparison.results[0][1].report.primary
|
|
66
|
+
for column in ("Model", primary, "Valid output rate", "Errors", "Cost", "Mean latency"):
|
|
67
|
+
table.add_column(column)
|
|
68
|
+
for name, record in comparison.results:
|
|
69
|
+
validity = record.report.values.get("valid_output_rate")
|
|
70
|
+
cost = "unavailable" if record.usage.cost_usd is None else f"${record.usage.cost_usd:.4f}"
|
|
71
|
+
latency = (
|
|
72
|
+
fmt(record.usage.mean_latency_s, 2) + "s" if record.usage.mean_latency_s else "n/a"
|
|
73
|
+
)
|
|
74
|
+
table.add_row(
|
|
75
|
+
name,
|
|
76
|
+
_value(record),
|
|
77
|
+
fmt(validity.value if validity else None),
|
|
78
|
+
f"{record.error_count}/{record.sample_count}",
|
|
79
|
+
cost,
|
|
80
|
+
latency,
|
|
81
|
+
)
|
|
82
|
+
console.print(table)
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def _value(record: EvaluationRecord) -> str:
|
|
86
|
+
return fmt(record.report.primary_value.value)
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def final_test_payload(result: FinalTest) -> dict[str, Any]:
|
|
90
|
+
ref = result.version.ref
|
|
91
|
+
return {
|
|
92
|
+
"version": ref,
|
|
93
|
+
"test": record_payload(result.record, ref),
|
|
94
|
+
"validation": record_payload(result.validation, ref) if result.validation else None,
|
|
95
|
+
"initial": (
|
|
96
|
+
record_payload(result.baseline[1], result.baseline[0].ref) if result.baseline else None
|
|
97
|
+
),
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def comparison_payload(comparison: Comparison) -> dict[str, Any]:
|
|
102
|
+
ref = comparison.version.ref
|
|
103
|
+
return {
|
|
104
|
+
"version": ref,
|
|
105
|
+
"partition": comparison.partition.value,
|
|
106
|
+
"models": [
|
|
107
|
+
{"model": name, **record_payload(record, ref)} for name, record in comparison.results
|
|
108
|
+
],
|
|
109
|
+
}
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
"""Exporting a stored prompt version as Markdown, plain text, JSON, or Strop prompt YAML."""
|
|
2
|
+
|
|
3
|
+
import json
|
|
4
|
+
from enum import StrEnum
|
|
5
|
+
|
|
6
|
+
from strop.prompts import dump_prompt
|
|
7
|
+
from strop.prompts.versions import PromptVersion
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class PromptFormat(StrEnum):
|
|
11
|
+
MD = "md"
|
|
12
|
+
TXT = "txt"
|
|
13
|
+
JSON = "json"
|
|
14
|
+
YAML = "yaml"
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def render_prompt(version: PromptVersion, fmt: PromptFormat) -> str:
|
|
18
|
+
"""The prompt messages exactly as stored, in the requested format."""
|
|
19
|
+
if fmt is PromptFormat.YAML:
|
|
20
|
+
return dump_prompt(version.template)
|
|
21
|
+
if fmt is PromptFormat.JSON:
|
|
22
|
+
return json.dumps(prompt_payload(version), indent=2, ensure_ascii=False) + "\n"
|
|
23
|
+
if fmt is PromptFormat.TXT:
|
|
24
|
+
return "\n\n".join(f"[{m.role}]\n{m.content}" for m in version.template.messages) + "\n"
|
|
25
|
+
return _markdown(version)
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def prompt_payload(version: PromptVersion) -> dict[str, object]:
|
|
29
|
+
template = version.template
|
|
30
|
+
return {
|
|
31
|
+
"version": version.number,
|
|
32
|
+
"parent": version.parent_number,
|
|
33
|
+
"source": version.source,
|
|
34
|
+
"change_summary": version.change_summary,
|
|
35
|
+
"variables": list(template.variables),
|
|
36
|
+
"messages": [
|
|
37
|
+
{"role": m.role, "content": m.content, "editable": template.is_editable(m)}
|
|
38
|
+
for m in template.messages
|
|
39
|
+
],
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def _markdown(version: PromptVersion) -> str:
|
|
44
|
+
template = version.template
|
|
45
|
+
lines = [f"# Prompt {version.ref}", ""]
|
|
46
|
+
if version.parent_number:
|
|
47
|
+
lines.append(f"- Parent: v{version.parent_number}")
|
|
48
|
+
lines.append(f"- Source: {version.source}")
|
|
49
|
+
if template.variables:
|
|
50
|
+
lines.append("- Variables: " + ", ".join(f"`{{{{{v}}}}}`" for v in template.variables))
|
|
51
|
+
if version.change_summary:
|
|
52
|
+
lines.append(f"- Summary: {version.change_summary}")
|
|
53
|
+
for index, message in enumerate(template.messages, 1):
|
|
54
|
+
state = "editable" if template.is_editable(message) else "frozen"
|
|
55
|
+
fence = "````" if "```" in message.content else "```"
|
|
56
|
+
lines += ["", f"## {message.role} (message {index}, {state})", "", f"{fence}text"]
|
|
57
|
+
lines += [message.content.rstrip("\n"), fence]
|
|
58
|
+
return "\n".join(lines) + "\n"
|