strop-cli 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (137) hide show
  1. strop/__init__.py +5 -0
  2. strop/__main__.py +5 -0
  3. strop/analysis/__init__.py +17 -0
  4. strop/analysis/classify.py +52 -0
  5. strop/analysis/evidence.py +52 -0
  6. strop/analysis/facts.py +74 -0
  7. strop/analysis/patterns.py +90 -0
  8. strop/cli/__init__.py +5 -0
  9. strop/cli/analyze.py +93 -0
  10. strop/cli/app.py +82 -0
  11. strop/cli/data.py +111 -0
  12. strop/cli/evaluate.py +145 -0
  13. strop/cli/export.py +74 -0
  14. strop/cli/final.py +106 -0
  15. strop/cli/init.py +131 -0
  16. strop/cli/optimize.py +156 -0
  17. strop/cli/options.py +20 -0
  18. strop/cli/progress.py +18 -0
  19. strop/cli/prompts.py +110 -0
  20. strop/cli/status.py +68 -0
  21. strop/cli/versions.py +123 -0
  22. strop/config/__init__.py +16 -0
  23. strop/config/base.py +9 -0
  24. strop/config/constraints.py +33 -0
  25. strop/config/dataset.py +63 -0
  26. strop/config/enums.py +52 -0
  27. strop/config/io.py +52 -0
  28. strop/config/limits.py +19 -0
  29. strop/config/llm.py +42 -0
  30. strop/config/metrics.py +86 -0
  31. strop/config/optimization.py +23 -0
  32. strop/config/output.py +31 -0
  33. strop/config/overrides.py +22 -0
  34. strop/config/root.py +36 -0
  35. strop/config/rules.py +67 -0
  36. strop/config/split.py +23 -0
  37. strop/data/__init__.py +58 -0
  38. strop/data/loaders.py +85 -0
  39. strop/data/models.py +44 -0
  40. strop/data/rows.py +99 -0
  41. strop/data/splitting.py +103 -0
  42. strop/env.py +44 -0
  43. strop/errors.py +59 -0
  44. strop/evaluation/__init__.py +1 -0
  45. strop/evaluation/budget.py +67 -0
  46. strop/evaluation/budgeted.py +23 -0
  47. strop/evaluation/examples.py +49 -0
  48. strop/evaluation/models.py +105 -0
  49. strop/evaluation/parsing.py +96 -0
  50. strop/evaluation/runner.py +107 -0
  51. strop/formatting.py +14 -0
  52. strop/llm/__init__.py +7 -0
  53. strop/llm/caching.py +28 -0
  54. strop/llm/client.py +15 -0
  55. strop/llm/factory.py +14 -0
  56. strop/llm/litellm_client.py +90 -0
  57. strop/llm/mock.py +43 -0
  58. strop/llm/retry.py +44 -0
  59. strop/llm/structured.py +52 -0
  60. strop/llm/types.py +59 -0
  61. strop/log.py +49 -0
  62. strop/metrics/__init__.py +1 -0
  63. strop/metrics/basic.py +30 -0
  64. strop/metrics/classification.py +164 -0
  65. strop/metrics/judge.py +130 -0
  66. strop/metrics/python_metric.py +101 -0
  67. strop/metrics/python_runner.py +65 -0
  68. strop/metrics/registry.py +115 -0
  69. strop/metrics/rules.py +55 -0
  70. strop/metrics/structured.py +109 -0
  71. strop/metrics/suite.py +109 -0
  72. strop/metrics/text.py +55 -0
  73. strop/metrics/types.py +106 -0
  74. strop/optimization/__init__.py +25 -0
  75. strop/optimization/acceptance.py +170 -0
  76. strop/optimization/generator.py +92 -0
  77. strop/optimization/meta_prompt.py +101 -0
  78. strop/optimization/types.py +69 -0
  79. strop/prompts/__init__.py +18 -0
  80. strop/prompts/diff.py +139 -0
  81. strop/prompts/files.py +92 -0
  82. strop/prompts/models.py +51 -0
  83. strop/prompts/render.py +16 -0
  84. strop/prompts/template.py +45 -0
  85. strop/prompts/validation.py +31 -0
  86. strop/prompts/versions.py +34 -0
  87. strop/py.typed +0 -0
  88. strop/reporting/__init__.py +6 -0
  89. strop/reporting/console.py +20 -0
  90. strop/reporting/diff_view.py +102 -0
  91. strop/reporting/evaluation_view.py +141 -0
  92. strop/reporting/final_view.py +109 -0
  93. strop/reporting/prompt_export.py +58 -0
  94. strop/reporting/report_markdown.py +157 -0
  95. strop/reporting/run_view.py +118 -0
  96. strop/services/__init__.py +1 -0
  97. strop/services/analysis.py +124 -0
  98. strop/services/clean.py +38 -0
  99. strop/services/compare.py +60 -0
  100. strop/services/context.py +70 -0
  101. strop/services/data.py +65 -0
  102. strop/services/evaluate.py +105 -0
  103. strop/services/evaluator.py +169 -0
  104. strop/services/export.py +90 -0
  105. strop/services/final_test.py +69 -0
  106. strop/services/metrics.py +89 -0
  107. strop/services/optimize.py +187 -0
  108. strop/services/partitions.py +46 -0
  109. strop/services/project.py +55 -0
  110. strop/services/prompts.py +76 -0
  111. strop/services/report.py +194 -0
  112. strop/services/review.py +43 -0
  113. strop/services/scaffold.py +79 -0
  114. strop/services/turn.py +234 -0
  115. strop/storage/__init__.py +6 -0
  116. strop/storage/cache.py +45 -0
  117. strop/storage/database.py +45 -0
  118. strop/storage/datasets.py +129 -0
  119. strop/storage/evaluations.py +160 -0
  120. strop/storage/migrations/001_initial.sql +5 -0
  121. strop/storage/migrations/002_datasets.sql +27 -0
  122. strop/storage/migrations/003_prompt_versions.sql +22 -0
  123. strop/storage/migrations/004_llm_cache.sql +6 -0
  124. strop/storage/migrations/005_evaluations.sql +37 -0
  125. strop/storage/migrations/006_runs.sql +52 -0
  126. strop/storage/migrations/007_example_feedback.sql +2 -0
  127. strop/storage/migrations/__init__.py +5 -0
  128. strop/storage/migrator.py +91 -0
  129. strop/storage/paths.py +30 -0
  130. strop/storage/prompts.py +96 -0
  131. strop/storage/run_models.py +75 -0
  132. strop/storage/runs.py +263 -0
  133. strop_cli-0.1.0.dist-info/METADATA +242 -0
  134. strop_cli-0.1.0.dist-info/RECORD +137 -0
  135. strop_cli-0.1.0.dist-info/WHEEL +4 -0
  136. strop_cli-0.1.0.dist-info/entry_points.txt +2 -0
  137. strop_cli-0.1.0.dist-info/licenses/LICENSE +21 -0
@@ -0,0 +1,51 @@
1
+ """Prompt template types."""
2
+
3
+ import hashlib
4
+ import json
5
+ from dataclasses import dataclass
6
+
7
+ from strop.llm.types import Role
8
+ from strop.prompts.template import find_variables
9
+
10
+
11
+ @dataclass(frozen=True, slots=True)
12
+ class PromptMessage:
13
+ role: Role
14
+ content: str
15
+
16
+
17
+ @dataclass(frozen=True, slots=True)
18
+ class PromptTemplate:
19
+ """Chat messages plus the roles whose messages the optimizer may rewrite."""
20
+
21
+ messages: tuple[PromptMessage, ...]
22
+ optimize: frozenset[Role]
23
+
24
+ @property
25
+ def variables(self) -> tuple[str, ...]:
26
+ """Variable names used anywhere in the template, in order of first use."""
27
+ names: dict[str, None] = {}
28
+ for message in self.messages:
29
+ names.update(dict.fromkeys(find_variables(message.content)))
30
+ return tuple(names)
31
+
32
+ def is_editable(self, message: PromptMessage) -> bool:
33
+ return message.role in self.optimize
34
+
35
+ def to_json(self) -> str:
36
+ """Canonical JSON used for storage and hashing."""
37
+ payload = {
38
+ "messages": [{"role": m.role, "content": m.content} for m in self.messages],
39
+ "optimize": sorted(self.optimize),
40
+ }
41
+ return json.dumps(payload, sort_keys=True, ensure_ascii=False)
42
+
43
+ @classmethod
44
+ def from_json(cls, text: str) -> "PromptTemplate":
45
+ data = json.loads(text)
46
+ messages = tuple(PromptMessage(m["role"], m["content"]) for m in data["messages"])
47
+ return cls(messages, frozenset(data["optimize"]))
48
+
49
+ @property
50
+ def content_hash(self) -> str:
51
+ return hashlib.sha256(self.to_json().encode("utf-8")).hexdigest()
@@ -0,0 +1,16 @@
1
+ """Rendering a template for one dataset row."""
2
+
3
+ from collections.abc import Mapping
4
+
5
+ from strop.errors import ConfigError
6
+ from strop.llm.types import Message
7
+ from strop.prompts.models import PromptTemplate
8
+ from strop.prompts.template import substitute
9
+
10
+
11
+ def render_messages(template: PromptTemplate, values: Mapping[str, str]) -> list[Message]:
12
+ """Fill every placeholder from ``values`` and return chat messages for the model."""
13
+ missing = [name for name in template.variables if name not in values]
14
+ if missing:
15
+ raise ConfigError(f"no value for prompt variable(s): {', '.join(missing)}")
16
+ return [Message(role=m.role, content=substitute(m.content, values)) for m in template.messages]
@@ -0,0 +1,45 @@
1
+ """Variable discovery and substitution for ``{{name}}`` placeholders."""
2
+
3
+ import re
4
+ from collections.abc import Mapping
5
+
6
+ from strop.errors import ConfigError
7
+
8
+ _VARIABLE = re.compile(r"\{\{\s*([A-Za-z_]\w*)\s*\}\}")
9
+ _FIXED = re.compile(r"<strop:fixed>(.*?)</strop:fixed>", re.DOTALL)
10
+ FIXED_OPEN = "<strop:fixed>"
11
+ FIXED_CLOSE = "</strop:fixed>"
12
+
13
+
14
+ def find_variables(text: str) -> list[str]:
15
+ """Distinct variable names in ``text``, in order of first use."""
16
+ return list(dict.fromkeys(_VARIABLE.findall(text)))
17
+
18
+
19
+ def fixed_spans(text: str) -> list[str]:
20
+ """Contents of every ``<strop:fixed>`` span, in order."""
21
+ return _FIXED.findall(text)
22
+
23
+
24
+ def strip_markers(text: str) -> str:
25
+ """Remove fixed-span markers, keeping their contents."""
26
+ return _FIXED.sub(lambda match: match.group(1), text)
27
+
28
+
29
+ def has_balanced_markers(text: str) -> bool:
30
+ return text.count(FIXED_OPEN) == text.count(FIXED_CLOSE) == len(_FIXED.findall(text))
31
+
32
+
33
+ def substitute(text: str, values: Mapping[str, str]) -> str:
34
+ """Replace placeholders in one pass, so substituted values are never expanded again."""
35
+ return _VARIABLE.sub(lambda match: values[match.group(1)], strip_markers(text))
36
+
37
+
38
+ def check_variables(used: tuple[str, ...], available: tuple[str, ...]) -> None:
39
+ """Raise if the prompt uses a variable that no dataset input column provides."""
40
+ unknown = [name for name in used if name not in available]
41
+ if unknown:
42
+ raise ConfigError(
43
+ f"the prompt uses variables with no matching input column: {', '.join(unknown)}; "
44
+ f"input columns: {', '.join(available)}"
45
+ )
@@ -0,0 +1,31 @@
1
+ """Checks that an optimizer-proposed revision only changed what it was allowed to change."""
2
+
3
+ from collections import Counter
4
+
5
+ from strop.prompts.models import PromptTemplate
6
+ from strop.prompts.template import fixed_spans
7
+
8
+
9
+ def validate_revision(parent: PromptTemplate, candidate: PromptTemplate) -> list[str]:
10
+ """Return a list of rule violations; an empty list means the revision is acceptable."""
11
+ if [m.role for m in parent.messages] != [m.role for m in candidate.messages]:
12
+ return ["the message structure (roles and order) must not change"]
13
+ problems = []
14
+ if parent.optimize != candidate.optimize:
15
+ problems.append("the set of editable roles must not change")
16
+ for index, (old, new) in enumerate(zip(parent.messages, candidate.messages, strict=True), 1):
17
+ label = f"message {index} ({old.role})"
18
+ if not parent.is_editable(old):
19
+ if old.content != new.content:
20
+ problems.append(f"{label} is frozen but was modified")
21
+ continue
22
+ if not new.content.strip():
23
+ problems.append(f"{label} is empty")
24
+ if Counter(fixed_spans(old.content)) != Counter(fixed_spans(new.content)):
25
+ problems.append(f"{label} changed or removed a <strop:fixed> span")
26
+ if set(parent.variables) != set(candidate.variables):
27
+ problems.append(
28
+ f"template variables changed: expected {sorted(parent.variables)}, "
29
+ f"got {sorted(candidate.variables)}"
30
+ )
31
+ return problems
@@ -0,0 +1,34 @@
1
+ """Prompt version records."""
2
+
3
+ from dataclasses import dataclass
4
+ from typing import Literal
5
+
6
+ from strop.errors import ConfigError
7
+ from strop.prompts.models import PromptTemplate
8
+
9
+ Source = Literal["user", "optimizer"]
10
+
11
+
12
+ @dataclass(frozen=True, slots=True)
13
+ class PromptVersion:
14
+ """One immutable revision of the prompt, identified as ``v<number>``."""
15
+
16
+ number: int
17
+ parent_number: int | None
18
+ template: PromptTemplate
19
+ change_summary: str
20
+ reason: str
21
+ source: Source
22
+ created_at: str
23
+
24
+ @property
25
+ def ref(self) -> str:
26
+ return f"v{self.number}"
27
+
28
+
29
+ def parse_version_ref(text: str) -> int:
30
+ """Parse ``v3`` or ``3`` into a version number."""
31
+ digits = text.strip().lower().removeprefix("v")
32
+ if not digits.isdigit() or int(digits) < 1:
33
+ raise ConfigError(f"invalid prompt version '{text}'; use a form like v3")
34
+ return int(digits)
strop/py.typed ADDED
File without changes
@@ -0,0 +1,6 @@
1
+ """Terminal and file output."""
2
+
3
+ from strop.reporting.console import ColorMode, make_console
4
+ from strop.reporting.diff_view import render_diff
5
+
6
+ __all__ = ["ColorMode", "make_console", "render_diff"]
@@ -0,0 +1,20 @@
1
+ """Console construction with an explicit color mode."""
2
+
3
+ from enum import StrEnum
4
+
5
+ from rich.console import Console
6
+
7
+
8
+ class ColorMode(StrEnum):
9
+ AUTO = "auto"
10
+ ALWAYS = "always"
11
+ NEVER = "never"
12
+
13
+
14
+ def make_console(mode: ColorMode = ColorMode.AUTO) -> Console:
15
+ """A stdout console. ``auto`` colors only on a terminal and respects ``NO_COLOR``."""
16
+ if mode is ColorMode.ALWAYS:
17
+ return Console(force_terminal=True, color_system="standard", highlight=False)
18
+ if mode is ColorMode.NEVER:
19
+ return Console(force_terminal=False, no_color=True, color_system=None, highlight=False)
20
+ return Console(highlight=False)
@@ -0,0 +1,102 @@
1
+ """Colored terminal rendering of a prompt diff.
2
+
3
+ Without color (pipes, files, ``NO_COLOR``) the output is a plain unified diff.
4
+ """
5
+
6
+ import difflib
7
+ import re
8
+
9
+ from rich.console import Console
10
+ from rich.text import Text
11
+
12
+ from strop.prompts.diff import DiffLine, MessageDiff, PromptDiff
13
+
14
+ _TOKEN = re.compile(r"\s+|\w+|[^\w\s]")
15
+ _STYLES = {
16
+ "remove": ("red", "bold red reverse"),
17
+ "add": ("green", "bold green reverse"),
18
+ }
19
+
20
+
21
+ def render_diff(diff: PromptDiff, console: Console) -> None:
22
+ """Print ``diff`` with colored lines, word-level highlights, and a summary line."""
23
+ for message in diff.messages:
24
+ console.print(_heading(message), soft_wrap=True)
25
+ for text in _message_lines(message.lines):
26
+ console.print(text, soft_wrap=True)
27
+ console.print(Text(_summary(diff), style="dim"), soft_wrap=True)
28
+
29
+
30
+ def _heading(message: MessageDiff) -> Text:
31
+ style = "bold" if message.status in {"changed", "added", "removed"} else "dim"
32
+ return Text(f"== message {message.index} ({message.role}): {message.status}", style=style)
33
+
34
+
35
+ def _message_lines(lines: tuple[DiffLine, ...]) -> list[Text]:
36
+ out: list[Text] = []
37
+ index = 0
38
+ while index < len(lines):
39
+ removed, added, index = _take_change_block(lines, index)
40
+ if removed or added:
41
+ out.extend(_change_block(removed, added))
42
+ continue
43
+ line = lines[index]
44
+ out.append(_plain_line(line))
45
+ index += 1
46
+ return out
47
+
48
+
49
+ def _take_change_block(lines: tuple[DiffLine, ...], start: int) -> tuple[list[str], list[str], int]:
50
+ index = start
51
+ removed: list[str] = []
52
+ added: list[str] = []
53
+ while index < len(lines) and lines[index].kind == "remove":
54
+ removed.append(lines[index].text)
55
+ index += 1
56
+ while index < len(lines) and lines[index].kind == "add":
57
+ added.append(lines[index].text)
58
+ index += 1
59
+ return removed, added, index
60
+
61
+
62
+ def _change_block(removed: list[str], added: list[str]) -> list[Text]:
63
+ old_texts = [Text("-")] * len(removed)
64
+ new_texts = [Text("+")] * len(added)
65
+ for position in range(max(len(removed), len(added))):
66
+ before = removed[position] if position < len(removed) else None
67
+ after = added[position] if position < len(added) else None
68
+ if before is not None and after is not None:
69
+ old_texts[position], new_texts[position] = _word_diff(before, after)
70
+ elif before is not None:
71
+ old_texts[position] = Text(f"-{before}", style=_STYLES["remove"][0])
72
+ elif after is not None:
73
+ new_texts[position] = Text(f"+{after}", style=_STYLES["add"][0])
74
+ return [*old_texts, *new_texts]
75
+
76
+
77
+ def _word_diff(before: str, after: str) -> tuple[Text, Text]:
78
+ old_tokens = _TOKEN.findall(before)
79
+ new_tokens = _TOKEN.findall(after)
80
+ old_text = Text("-", style=_STYLES["remove"][0])
81
+ new_text = Text("+", style=_STYLES["add"][0])
82
+ matcher = difflib.SequenceMatcher(None, old_tokens, new_tokens, autojunk=False)
83
+ for tag, i1, i2, j1, j2 in matcher.get_opcodes():
84
+ changed = tag != "equal"
85
+ old_text.append("".join(old_tokens[i1:i2]), _STYLES["remove"][1] if changed else None)
86
+ new_text.append("".join(new_tokens[j1:j2]), _STYLES["add"][1] if changed else None)
87
+ return old_text, new_text
88
+
89
+
90
+ def _plain_line(line: DiffLine) -> Text:
91
+ if line.kind == "hunk":
92
+ return Text(line.text, style="cyan")
93
+ return Text(f"{line.prefix}{line.text}", style="dim")
94
+
95
+
96
+ def _summary(diff: PromptDiff) -> str:
97
+ stats = diff.stats
98
+ change = stats.approx_token_change
99
+ return (
100
+ f"Summary: +{stats.added_lines} line(s), -{stats.removed_lines} line(s), "
101
+ f"prompt {stats.chars_before} -> {stats.chars_after} chars (~{change:+d} tokens)"
102
+ )
@@ -0,0 +1,141 @@
1
+ """Terminal rendering and JSON payloads for evaluations."""
2
+
3
+ from collections import Counter
4
+ from collections.abc import Sequence
5
+ from typing import Any
6
+
7
+ from rich.console import Console
8
+ from rich.table import Table
9
+ from rich.text import Text
10
+
11
+ from strop.evaluation.models import EvaluationRecord, ExampleOutcome, UsageSummary
12
+ from strop.formatting import fmt, shorten
13
+
14
+
15
+ def render_evaluation(
16
+ console: Console,
17
+ record: EvaluationRecord,
18
+ outcomes: Sequence[ExampleOutcome],
19
+ *,
20
+ version_ref: str,
21
+ show_errors: int = 5,
22
+ ) -> None:
23
+ """Print the metrics, error summary, per-class scores, usage, and sample failures."""
24
+ title = (
25
+ f"Evaluation {record.id}: {version_ref} on {record.partition.value} "
26
+ f"({record.sample_count} examples, {record.kind})"
27
+ )
28
+ console.print(Text(title, style="bold"), soft_wrap=True)
29
+ console.print(_metrics_table(record))
30
+ console.print(_error_summary(outcomes), soft_wrap=True)
31
+ classes = _per_class_table(record)
32
+ if classes is not None:
33
+ console.print(classes)
34
+ console.print(Text(usage_line(record.usage), style="dim"), soft_wrap=True)
35
+ for line in failure_lines(outcomes, show_errors):
36
+ console.print(line, soft_wrap=True, markup=False)
37
+
38
+
39
+ def _metrics_table(record: EvaluationRecord) -> Table:
40
+ table = Table(box=None, pad_edge=False, header_style="bold")
41
+ for column in ("Metric", "Value", "Samples", "Note"):
42
+ table.add_column(column)
43
+ for name, metric in record.report.values.items():
44
+ label = f"{name} (primary)" if name == record.report.primary else name
45
+ note = "subjective, LLM-judged" if metric.subjective else ""
46
+ table.add_row(label, fmt(metric.value), str(metric.samples), note)
47
+ return table
48
+
49
+
50
+ def _error_summary(outcomes: Sequence[ExampleOutcome]) -> str:
51
+ codes = Counter(o.error for o in outcomes if o.error)
52
+ wrong = sum(1 for o in outcomes if _is_wrong(o))
53
+ parts = [f"{code} {count}" for code, count in sorted(codes.items())]
54
+ if wrong:
55
+ parts.append(f"valid but wrong {wrong}")
56
+ return f"Output problems: {', '.join(parts)}" if parts else "No output problems."
57
+
58
+
59
+ def _is_wrong(outcome: ExampleOutcome) -> bool:
60
+ return (
61
+ outcome.value is not None
62
+ and outcome.expected is not None
63
+ and outcome.value != outcome.expected
64
+ )
65
+
66
+
67
+ def _per_class_table(record: EvaluationRecord) -> Table | None:
68
+ per_class = record.report.primary_value.details.get("per_class")
69
+ if not per_class:
70
+ return None
71
+ table = Table(
72
+ box=None, pad_edge=False, header_style="bold", title="Per class", title_justify="left"
73
+ )
74
+ for column in ("Class", "Precision", "Recall", "F1", "Support"):
75
+ table.add_column(column)
76
+ for label, scores in per_class.items():
77
+ table.add_row(
78
+ label,
79
+ fmt(scores["precision"]),
80
+ fmt(scores["recall"]),
81
+ fmt(scores["f1"]),
82
+ str(scores["support"]),
83
+ )
84
+ return table
85
+
86
+
87
+ def usage_line(usage: UsageSummary) -> str:
88
+ cost = "unavailable" if usage.cost_usd is None else f"${usage.cost_usd:.4f}"
89
+ if usage.cost_usd is not None and not usage.cost_complete:
90
+ cost += " (partial)"
91
+ tokens = "unavailable" if usage.total_tokens is None else f"{usage.total_tokens:,}"
92
+ latency = "unavailable" if usage.mean_latency_s is None else f"{usage.mean_latency_s:.2f}s"
93
+ return (
94
+ f"Calls: {usage.calls} ({usage.cached_calls} cached) | Tokens: {tokens} | "
95
+ f"Cost: {cost} | Mean latency: {latency}"
96
+ )
97
+
98
+
99
+ def failure_lines(outcomes: Sequence[ExampleOutcome], limit: int) -> list[str]:
100
+ """One line per failing example, up to ``limit``, with a trailer for the rest."""
101
+ failing = [o for o in outcomes if o.error or _is_wrong(o)]
102
+ if limit <= 0 or not failing:
103
+ return []
104
+ lines = ["Examples that failed:"]
105
+ for outcome in failing[:limit]:
106
+ if outcome.error:
107
+ got = shorten(outcome.detail or outcome.raw_output)
108
+ lines.append(f" {outcome.row_id}: {outcome.error} ({got})")
109
+ else:
110
+ lines.append(
111
+ f" {outcome.row_id}: expected {shorten(outcome.expected or '')!r}, "
112
+ f"got {shorten(outcome.value or '')!r}"
113
+ )
114
+ if len(failing) > limit:
115
+ lines.append(f" ... and {len(failing) - limit} more")
116
+ return lines
117
+
118
+
119
+ def record_payload(record: EvaluationRecord, version_ref: str) -> dict[str, Any]:
120
+ usage = record.usage
121
+ return {
122
+ "evaluation": record.id,
123
+ "version": version_ref,
124
+ "partition": record.partition.value,
125
+ "kind": record.kind,
126
+ "samples": record.sample_count,
127
+ "errors": record.error_count,
128
+ "primary_metric": record.report.primary,
129
+ "metrics": {
130
+ name: {"value": m.value, "samples": m.samples, "subjective": m.subjective}
131
+ for name, m in record.report.values.items()
132
+ },
133
+ "usage": {
134
+ "calls": usage.calls,
135
+ "cached_calls": usage.cached_calls,
136
+ "total_tokens": usage.total_tokens,
137
+ "cost_usd": usage.cost_usd,
138
+ "cost_complete": usage.cost_complete,
139
+ "mean_latency_s": usage.mean_latency_s,
140
+ },
141
+ }
@@ -0,0 +1,109 @@
1
+ """Terminal rendering and JSON payloads for the test and compare commands."""
2
+
3
+ import math
4
+ from typing import Any
5
+
6
+ from rich.console import Console
7
+ from rich.table import Table
8
+ from rich.text import Text
9
+
10
+ from strop.evaluation.models import EvaluationRecord
11
+ from strop.formatting import fmt
12
+ from strop.reporting.evaluation_view import record_payload, usage_line
13
+ from strop.services.compare import Comparison
14
+ from strop.services.final_test import FinalTest
15
+
16
+
17
+ def render_final_test(console: Console, result: FinalTest) -> None:
18
+ """Summarize the held-out result next to the validation result it should be read against."""
19
+ record = result.record
20
+ primary = record.report.primary
21
+ console.print(Text("Held-out test result", style="bold"), soft_wrap=True)
22
+ table = Table(box=None, pad_edge=False, header_style="bold")
23
+ for column in ("Prompt", "Partition", primary, "Examples"):
24
+ table.add_column(column)
25
+ if result.baseline is not None:
26
+ initial, base = result.baseline
27
+ table.add_row(initial.ref + " (initial)", "test", _value(base), str(base.sample_count))
28
+ if result.validation is not None:
29
+ table.add_row(
30
+ result.version.ref,
31
+ "validation",
32
+ _value(result.validation),
33
+ str(result.validation.sample_count),
34
+ )
35
+ table.add_row(result.version.ref, "test", _value(record), str(record.sample_count))
36
+ console.print(table)
37
+ console.print(Text(usage_line(record.usage), style="dim"), soft_wrap=True)
38
+ note = standard_error_note(record)
39
+ if note:
40
+ console.print(note, soft_wrap=True, highlight=False)
41
+ console.print(
42
+ "Do not change the prompt based on this result: the test set is now used up.",
43
+ soft_wrap=True,
44
+ highlight=False,
45
+ )
46
+
47
+
48
+ def standard_error_note(record: EvaluationRecord) -> str | None:
49
+ """A rough noise estimate for rate-like metrics, so small test sets are not over-read."""
50
+ value = record.report.primary_value.value
51
+ n = record.report.primary_value.samples
52
+ if value is None or n == 0 or not 0.0 <= value <= 1.0:
53
+ return None
54
+ error = math.sqrt(value * (1 - value) / n)
55
+ return (
56
+ f"Note: with {n} test examples, differences smaller than about {error:.2f} "
57
+ f"(one standard error for a rate near {value:.2f}) may be noise."
58
+ )
59
+
60
+
61
+ def render_comparison(console: Console, comparison: Comparison) -> None:
62
+ title = f"{comparison.version.ref} on {comparison.partition.value}, same rows for every model"
63
+ console.print(Text(title, style="bold"), soft_wrap=True)
64
+ table = Table(box=None, pad_edge=False, header_style="bold")
65
+ primary = comparison.results[0][1].report.primary
66
+ for column in ("Model", primary, "Valid output rate", "Errors", "Cost", "Mean latency"):
67
+ table.add_column(column)
68
+ for name, record in comparison.results:
69
+ validity = record.report.values.get("valid_output_rate")
70
+ cost = "unavailable" if record.usage.cost_usd is None else f"${record.usage.cost_usd:.4f}"
71
+ latency = (
72
+ fmt(record.usage.mean_latency_s, 2) + "s" if record.usage.mean_latency_s else "n/a"
73
+ )
74
+ table.add_row(
75
+ name,
76
+ _value(record),
77
+ fmt(validity.value if validity else None),
78
+ f"{record.error_count}/{record.sample_count}",
79
+ cost,
80
+ latency,
81
+ )
82
+ console.print(table)
83
+
84
+
85
+ def _value(record: EvaluationRecord) -> str:
86
+ return fmt(record.report.primary_value.value)
87
+
88
+
89
+ def final_test_payload(result: FinalTest) -> dict[str, Any]:
90
+ ref = result.version.ref
91
+ return {
92
+ "version": ref,
93
+ "test": record_payload(result.record, ref),
94
+ "validation": record_payload(result.validation, ref) if result.validation else None,
95
+ "initial": (
96
+ record_payload(result.baseline[1], result.baseline[0].ref) if result.baseline else None
97
+ ),
98
+ }
99
+
100
+
101
+ def comparison_payload(comparison: Comparison) -> dict[str, Any]:
102
+ ref = comparison.version.ref
103
+ return {
104
+ "version": ref,
105
+ "partition": comparison.partition.value,
106
+ "models": [
107
+ {"model": name, **record_payload(record, ref)} for name, record in comparison.results
108
+ ],
109
+ }
@@ -0,0 +1,58 @@
1
+ """Exporting a stored prompt version as Markdown, plain text, JSON, or Strop prompt YAML."""
2
+
3
+ import json
4
+ from enum import StrEnum
5
+
6
+ from strop.prompts import dump_prompt
7
+ from strop.prompts.versions import PromptVersion
8
+
9
+
10
+ class PromptFormat(StrEnum):
11
+ MD = "md"
12
+ TXT = "txt"
13
+ JSON = "json"
14
+ YAML = "yaml"
15
+
16
+
17
+ def render_prompt(version: PromptVersion, fmt: PromptFormat) -> str:
18
+ """The prompt messages exactly as stored, in the requested format."""
19
+ if fmt is PromptFormat.YAML:
20
+ return dump_prompt(version.template)
21
+ if fmt is PromptFormat.JSON:
22
+ return json.dumps(prompt_payload(version), indent=2, ensure_ascii=False) + "\n"
23
+ if fmt is PromptFormat.TXT:
24
+ return "\n\n".join(f"[{m.role}]\n{m.content}" for m in version.template.messages) + "\n"
25
+ return _markdown(version)
26
+
27
+
28
+ def prompt_payload(version: PromptVersion) -> dict[str, object]:
29
+ template = version.template
30
+ return {
31
+ "version": version.number,
32
+ "parent": version.parent_number,
33
+ "source": version.source,
34
+ "change_summary": version.change_summary,
35
+ "variables": list(template.variables),
36
+ "messages": [
37
+ {"role": m.role, "content": m.content, "editable": template.is_editable(m)}
38
+ for m in template.messages
39
+ ],
40
+ }
41
+
42
+
43
+ def _markdown(version: PromptVersion) -> str:
44
+ template = version.template
45
+ lines = [f"# Prompt {version.ref}", ""]
46
+ if version.parent_number:
47
+ lines.append(f"- Parent: v{version.parent_number}")
48
+ lines.append(f"- Source: {version.source}")
49
+ if template.variables:
50
+ lines.append("- Variables: " + ", ".join(f"`{{{{{v}}}}}`" for v in template.variables))
51
+ if version.change_summary:
52
+ lines.append(f"- Summary: {version.change_summary}")
53
+ for index, message in enumerate(template.messages, 1):
54
+ state = "editable" if template.is_editable(message) else "frozen"
55
+ fence = "````" if "```" in message.content else "```"
56
+ lines += ["", f"## {message.role} (message {index}, {state})", "", f"{fence}text"]
57
+ lines += [message.content.rstrip("\n"), fence]
58
+ return "\n".join(lines) + "\n"