strop-cli 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (137) hide show
  1. strop/__init__.py +5 -0
  2. strop/__main__.py +5 -0
  3. strop/analysis/__init__.py +17 -0
  4. strop/analysis/classify.py +52 -0
  5. strop/analysis/evidence.py +52 -0
  6. strop/analysis/facts.py +74 -0
  7. strop/analysis/patterns.py +90 -0
  8. strop/cli/__init__.py +5 -0
  9. strop/cli/analyze.py +93 -0
  10. strop/cli/app.py +82 -0
  11. strop/cli/data.py +111 -0
  12. strop/cli/evaluate.py +145 -0
  13. strop/cli/export.py +74 -0
  14. strop/cli/final.py +106 -0
  15. strop/cli/init.py +131 -0
  16. strop/cli/optimize.py +156 -0
  17. strop/cli/options.py +20 -0
  18. strop/cli/progress.py +18 -0
  19. strop/cli/prompts.py +110 -0
  20. strop/cli/status.py +68 -0
  21. strop/cli/versions.py +123 -0
  22. strop/config/__init__.py +16 -0
  23. strop/config/base.py +9 -0
  24. strop/config/constraints.py +33 -0
  25. strop/config/dataset.py +63 -0
  26. strop/config/enums.py +52 -0
  27. strop/config/io.py +52 -0
  28. strop/config/limits.py +19 -0
  29. strop/config/llm.py +42 -0
  30. strop/config/metrics.py +86 -0
  31. strop/config/optimization.py +23 -0
  32. strop/config/output.py +31 -0
  33. strop/config/overrides.py +22 -0
  34. strop/config/root.py +36 -0
  35. strop/config/rules.py +67 -0
  36. strop/config/split.py +23 -0
  37. strop/data/__init__.py +58 -0
  38. strop/data/loaders.py +85 -0
  39. strop/data/models.py +44 -0
  40. strop/data/rows.py +99 -0
  41. strop/data/splitting.py +103 -0
  42. strop/env.py +44 -0
  43. strop/errors.py +59 -0
  44. strop/evaluation/__init__.py +1 -0
  45. strop/evaluation/budget.py +67 -0
  46. strop/evaluation/budgeted.py +23 -0
  47. strop/evaluation/examples.py +49 -0
  48. strop/evaluation/models.py +105 -0
  49. strop/evaluation/parsing.py +96 -0
  50. strop/evaluation/runner.py +107 -0
  51. strop/formatting.py +14 -0
  52. strop/llm/__init__.py +7 -0
  53. strop/llm/caching.py +28 -0
  54. strop/llm/client.py +15 -0
  55. strop/llm/factory.py +14 -0
  56. strop/llm/litellm_client.py +90 -0
  57. strop/llm/mock.py +43 -0
  58. strop/llm/retry.py +44 -0
  59. strop/llm/structured.py +52 -0
  60. strop/llm/types.py +59 -0
  61. strop/log.py +49 -0
  62. strop/metrics/__init__.py +1 -0
  63. strop/metrics/basic.py +30 -0
  64. strop/metrics/classification.py +164 -0
  65. strop/metrics/judge.py +130 -0
  66. strop/metrics/python_metric.py +101 -0
  67. strop/metrics/python_runner.py +65 -0
  68. strop/metrics/registry.py +115 -0
  69. strop/metrics/rules.py +55 -0
  70. strop/metrics/structured.py +109 -0
  71. strop/metrics/suite.py +109 -0
  72. strop/metrics/text.py +55 -0
  73. strop/metrics/types.py +106 -0
  74. strop/optimization/__init__.py +25 -0
  75. strop/optimization/acceptance.py +170 -0
  76. strop/optimization/generator.py +92 -0
  77. strop/optimization/meta_prompt.py +101 -0
  78. strop/optimization/types.py +69 -0
  79. strop/prompts/__init__.py +18 -0
  80. strop/prompts/diff.py +139 -0
  81. strop/prompts/files.py +92 -0
  82. strop/prompts/models.py +51 -0
  83. strop/prompts/render.py +16 -0
  84. strop/prompts/template.py +45 -0
  85. strop/prompts/validation.py +31 -0
  86. strop/prompts/versions.py +34 -0
  87. strop/py.typed +0 -0
  88. strop/reporting/__init__.py +6 -0
  89. strop/reporting/console.py +20 -0
  90. strop/reporting/diff_view.py +102 -0
  91. strop/reporting/evaluation_view.py +141 -0
  92. strop/reporting/final_view.py +109 -0
  93. strop/reporting/prompt_export.py +58 -0
  94. strop/reporting/report_markdown.py +157 -0
  95. strop/reporting/run_view.py +118 -0
  96. strop/services/__init__.py +1 -0
  97. strop/services/analysis.py +124 -0
  98. strop/services/clean.py +38 -0
  99. strop/services/compare.py +60 -0
  100. strop/services/context.py +70 -0
  101. strop/services/data.py +65 -0
  102. strop/services/evaluate.py +105 -0
  103. strop/services/evaluator.py +169 -0
  104. strop/services/export.py +90 -0
  105. strop/services/final_test.py +69 -0
  106. strop/services/metrics.py +89 -0
  107. strop/services/optimize.py +187 -0
  108. strop/services/partitions.py +46 -0
  109. strop/services/project.py +55 -0
  110. strop/services/prompts.py +76 -0
  111. strop/services/report.py +194 -0
  112. strop/services/review.py +43 -0
  113. strop/services/scaffold.py +79 -0
  114. strop/services/turn.py +234 -0
  115. strop/storage/__init__.py +6 -0
  116. strop/storage/cache.py +45 -0
  117. strop/storage/database.py +45 -0
  118. strop/storage/datasets.py +129 -0
  119. strop/storage/evaluations.py +160 -0
  120. strop/storage/migrations/001_initial.sql +5 -0
  121. strop/storage/migrations/002_datasets.sql +27 -0
  122. strop/storage/migrations/003_prompt_versions.sql +22 -0
  123. strop/storage/migrations/004_llm_cache.sql +6 -0
  124. strop/storage/migrations/005_evaluations.sql +37 -0
  125. strop/storage/migrations/006_runs.sql +52 -0
  126. strop/storage/migrations/007_example_feedback.sql +2 -0
  127. strop/storage/migrations/__init__.py +5 -0
  128. strop/storage/migrator.py +91 -0
  129. strop/storage/paths.py +30 -0
  130. strop/storage/prompts.py +96 -0
  131. strop/storage/run_models.py +75 -0
  132. strop/storage/runs.py +263 -0
  133. strop_cli-0.1.0.dist-info/METADATA +242 -0
  134. strop_cli-0.1.0.dist-info/RECORD +137 -0
  135. strop_cli-0.1.0.dist-info/WHEEL +4 -0
  136. strop_cli-0.1.0.dist-info/entry_points.txt +2 -0
  137. strop_cli-0.1.0.dist-info/licenses/LICENSE +21 -0
@@ -0,0 +1,65 @@
1
+ """Runs a user's metric file on a batch of examples.
2
+
3
+ This file is executed as a script in a separate Python process by ``python_metric``. It
4
+ imports nothing from Strop. The user's ``score(output, row)`` function may return a number
5
+ between 0 and 1, a bool, or a dict ``{"score": number, "feedback": "text"}``.
6
+ """
7
+
8
+ import importlib.util
9
+ import json
10
+ import sys
11
+ from pathlib import Path
12
+ from typing import Any
13
+
14
+
15
+ def _normalize(value: Any) -> dict[str, Any]:
16
+ if isinstance(value, bool):
17
+ return {"score": 1.0 if value else 0.0}
18
+ if isinstance(value, int | float):
19
+ return _checked(float(value), None)
20
+ if isinstance(value, dict) and isinstance(value.get("score"), int | float):
21
+ feedback = value.get("feedback")
22
+ return _checked(float(value["score"]), str(feedback) if feedback else None)
23
+ return {"error": "score() must return a number or a dict with a 'score' key"}
24
+
25
+
26
+ def _checked(score: float, feedback: str | None) -> dict[str, Any]:
27
+ if not 0.0 <= score <= 1.0:
28
+ return {"error": f"score {score} is outside the range 0 to 1"}
29
+ return {"score": score, "feedback": feedback}
30
+
31
+
32
+ def _load(path: str) -> Any:
33
+ spec = importlib.util.spec_from_file_location("strop_user_metric", path)
34
+ if spec is None or spec.loader is None:
35
+ raise ImportError(f"cannot load {path}")
36
+ module = importlib.util.module_from_spec(spec)
37
+ sys.path.insert(0, str(Path(path).resolve().parent))
38
+ spec.loader.exec_module(module)
39
+ if not callable(getattr(module, "score", None)):
40
+ raise AttributeError("the metric file must define a function score(output, row)")
41
+ return module.score
42
+
43
+
44
+ def main(path: str) -> int:
45
+ real_stdout = sys.stdout
46
+ sys.stdout = sys.stderr # user print() calls must not corrupt the JSON result
47
+ try:
48
+ score = _load(path)
49
+ except Exception as exc:
50
+ sys.stdout = real_stdout
51
+ json.dump({"fatal": f"{type(exc).__name__}: {exc}"}, sys.stdout)
52
+ return 3
53
+ results = []
54
+ for item in json.load(sys.stdin):
55
+ try:
56
+ results.append(_normalize(score(item["output"], item["row"])))
57
+ except Exception as exc:
58
+ results.append({"error": f"{type(exc).__name__}: {exc}"})
59
+ sys.stdout = real_stdout
60
+ json.dump({"results": results}, sys.stdout)
61
+ return 0
62
+
63
+
64
+ if __name__ == "__main__":
65
+ sys.exit(main(sys.argv[1]))
@@ -0,0 +1,115 @@
1
+ """Resolving metric names from strop.yaml to metric objects."""
2
+
3
+ from collections.abc import Callable, Mapping
4
+ from dataclasses import dataclass, field
5
+
6
+ from strop.config.metrics import RuleSpec
7
+ from strop.errors import ConfigError
8
+ from strop.metrics.basic import ExactMatch, ValidOutputRate
9
+ from strop.metrics.classification import Accuracy, Average, Averaged, ClassScore, Kind
10
+ from strop.metrics.rules import RuleMetric
11
+ from strop.metrics.structured import (
12
+ FieldAccuracy,
13
+ FieldCompleteness,
14
+ FieldF1,
15
+ JsonValid,
16
+ SchemaValid,
17
+ )
18
+ from strop.metrics.text import NumericClose, Similarity
19
+ from strop.metrics.types import Metric
20
+
21
+ _AVERAGES: tuple[Average, ...] = ("macro", "micro", "weighted")
22
+ _KINDS: tuple[Kind, ...] = ("precision", "recall", "f1")
23
+
24
+
25
+ @dataclass(frozen=True)
26
+ class MetricContext:
27
+ """What metric constructors need to know about the task."""
28
+
29
+ labels: tuple[str, ...] = ()
30
+ rules: Mapping[str, RuleSpec] = field(default_factory=dict)
31
+
32
+
33
+ Factory = Callable[[MetricContext], Metric]
34
+ ParamFactory = Callable[[str, MetricContext], Metric]
35
+
36
+
37
+ def _averaged_factory(average: Average, kind: Kind) -> Factory:
38
+ return lambda ctx: Averaged(average, kind, ctx.labels)
39
+
40
+
41
+ def _simple_factories() -> dict[str, Factory]:
42
+ factories: dict[str, Factory] = {
43
+ "accuracy": lambda ctx: Accuracy(ctx.labels),
44
+ "exact_match": lambda ctx: ExactMatch(),
45
+ "valid_output_rate": lambda ctx: ValidOutputRate(),
46
+ "json_valid": lambda ctx: JsonValid(),
47
+ "schema_valid": lambda ctx: SchemaValid(),
48
+ "field_completeness": lambda ctx: FieldCompleteness(),
49
+ "field_accuracy": lambda ctx: FieldAccuracy(),
50
+ "field_f1": lambda ctx: FieldF1(),
51
+ "similarity": lambda ctx: Similarity(),
52
+ "numeric_close": lambda ctx: NumericClose(),
53
+ }
54
+ for average in _AVERAGES:
55
+ for kind in _KINDS:
56
+ factories[f"{average}_{kind}"] = _averaged_factory(average, kind)
57
+ return factories
58
+
59
+
60
+ def _class_factory(kind: Kind) -> ParamFactory:
61
+ def build(label: str, ctx: MetricContext) -> Metric:
62
+ if ctx.labels and label not in ctx.labels:
63
+ raise ConfigError(
64
+ f"metric '{kind}:{label}' names an unknown label; labels: {', '.join(ctx.labels)}"
65
+ )
66
+ return ClassScore(kind, label, ctx.labels)
67
+
68
+ return build
69
+
70
+
71
+ def _rule_factory(name: str, ctx: MetricContext) -> Metric:
72
+ if name not in ctx.rules:
73
+ defined = ", ".join(ctx.rules) or "none"
74
+ raise ConfigError(f"rule '{name}' is not defined under metrics.rules; defined: {defined}")
75
+ return RuleMetric(name, ctx.rules[name])
76
+
77
+
78
+ def _numeric_close_factory(tolerance: str, ctx: MetricContext) -> Metric:
79
+ try:
80
+ return NumericClose(float(tolerance))
81
+ except ValueError:
82
+ raise ConfigError(f"metric 'numeric_close:{tolerance}' needs a numeric tolerance") from None
83
+
84
+
85
+ _SIMPLE = _simple_factories()
86
+ _PARAMETERIZED: dict[str, ParamFactory] = {
87
+ **{kind: _class_factory(kind) for kind in _KINDS},
88
+ "rule": _rule_factory,
89
+ "numeric_close": _numeric_close_factory,
90
+ }
91
+
92
+
93
+ def known_metric_names() -> list[str]:
94
+ """Built-in metric names, plus the parameterized forms."""
95
+ return sorted(_SIMPLE) + [f"{prefix}:<label>" for prefix in sorted(_PARAMETERIZED)]
96
+
97
+
98
+ def register_parameterized(prefix: str, factory: ParamFactory) -> None:
99
+ """Add a metric family addressed as ``<prefix>:<argument>``."""
100
+ _PARAMETERIZED[prefix] = factory
101
+
102
+
103
+ def register(name: str, factory: Factory) -> None:
104
+ """Add a built-in metric addressed by ``name``."""
105
+ _SIMPLE[name] = factory
106
+
107
+
108
+ def build_metric(name: str, context: MetricContext) -> Metric:
109
+ """Create the metric called ``name`` or raise ``ConfigError`` listing the valid names."""
110
+ if name in _SIMPLE:
111
+ return _SIMPLE[name](context)
112
+ prefix, separator, argument = name.partition(":")
113
+ if separator and argument and prefix in _PARAMETERIZED:
114
+ return _PARAMETERIZED[prefix](argument, context)
115
+ raise ConfigError(f"unknown metric '{name}'; available: {', '.join(known_metric_names())}")
strop/metrics/rules.py ADDED
@@ -0,0 +1,55 @@
1
+ """Rule checks: deterministic pass/fail tests on each output, reported as pass rates."""
2
+
3
+ import re
4
+ from collections.abc import Callable, Sequence
5
+
6
+ from strop.config.enums import RuleType
7
+ from strop.config.metrics import RuleSpec
8
+ from strop.metrics.types import ErrorCode, Example, Metric, MetricValue
9
+
10
+ Check = Callable[[str, RuleSpec], bool]
11
+
12
+ _CHECKS: dict[RuleType, Check] = {
13
+ RuleType.MAX_CHARS: lambda text, spec: len(text) <= int(str(spec.value)),
14
+ RuleType.MIN_CHARS: lambda text, spec: len(text) >= int(str(spec.value)),
15
+ RuleType.MAX_WORDS: lambda text, spec: len(text.split()) <= int(str(spec.value)),
16
+ RuleType.MIN_WORDS: lambda text, spec: len(text.split()) >= int(str(spec.value)),
17
+ RuleType.MATCHES: lambda text, spec: re.search(str(spec.value), text) is not None,
18
+ RuleType.NOT_MATCHES: lambda text, spec: re.search(str(spec.value), text) is None,
19
+ RuleType.CONTAINS_ALL: lambda text, spec: all(
20
+ term.casefold() in text.casefold() for term in _terms(spec)
21
+ ),
22
+ RuleType.CONTAINS_NONE: lambda text, spec: (
23
+ not any(term.casefold() in text.casefold() for term in _terms(spec))
24
+ ),
25
+ }
26
+
27
+
28
+ def _terms(spec: RuleSpec) -> list[str]:
29
+ return spec.value if isinstance(spec.value, list) else [str(spec.value)]
30
+
31
+
32
+ class RuleMetric(Metric):
33
+ """Pass rate of one rule, named ``rule:<name>``.
34
+
35
+ Outputs lost to provider failures are not scored; every other output, including an
36
+ empty one, is checked as written.
37
+ """
38
+
39
+ needs_expected = False
40
+
41
+ def __init__(self, rule_name: str, spec: RuleSpec) -> None:
42
+ self.name = f"rule:{rule_name}"
43
+ self._spec = spec
44
+
45
+ def compute(self, examples: Sequence[Example]) -> MetricValue:
46
+ scores = [s for e in examples if (s := self.per_example(e)) is not None]
47
+ return self._result(sum(scores) / len(scores) if scores else None, len(scores))
48
+
49
+ def per_example(self, example: Example) -> float | None:
50
+ if example.error == ErrorCode.PROVIDER_ERROR:
51
+ return None
52
+ text = example.raw_output.strip()
53
+ if not text:
54
+ return 0.0
55
+ return 1.0 if _CHECKS[self._spec.type](text, self._spec) else 0.0
@@ -0,0 +1,109 @@
1
+ """Metrics for JSON outputs: validity and field-level comparison against the expected object."""
2
+
3
+ import json
4
+ from collections.abc import Sequence
5
+ from typing import Any
6
+
7
+ from strop.metrics.types import ErrorCode, Example, Metric, MetricValue
8
+
9
+
10
+ def _canonical(value: Any) -> str:
11
+ return json.dumps(value, sort_keys=True, ensure_ascii=False)
12
+
13
+
14
+ def _expected_fields(example: Example) -> dict[str, str] | None:
15
+ expected = example.expected_json
16
+ if not isinstance(expected, dict):
17
+ return None
18
+ return {key: _canonical(value) for key, value in expected.items()}
19
+
20
+
21
+ def _predicted_fields(example: Example) -> dict[str, str]:
22
+ if not isinstance(example.parsed, dict):
23
+ return {}
24
+ return {key: _canonical(value) for key, value in example.parsed.items()}
25
+
26
+
27
+ def _mean(scores: list[float]) -> float | None:
28
+ return sum(scores) / len(scores) if scores else None
29
+
30
+
31
+ class JsonValid(Metric):
32
+ """Share of outputs that parse as JSON, before any schema check."""
33
+
34
+ name = "json_valid"
35
+ needs_expected = False
36
+
37
+ def compute(self, examples: Sequence[Example]) -> MetricValue:
38
+ value = _mean([self.per_example(e) or 0.0 for e in examples])
39
+ return self._result(value, len(examples))
40
+
41
+ def per_example(self, example: Example) -> float | None:
42
+ return 1.0 if example.parsed is not None else 0.0
43
+
44
+
45
+ class SchemaValid(Metric):
46
+ """Share of outputs that parse as JSON and satisfy the configured schema."""
47
+
48
+ name = "schema_valid"
49
+ needs_expected = False
50
+
51
+ def compute(self, examples: Sequence[Example]) -> MetricValue:
52
+ value = _mean([self.per_example(e) or 0.0 for e in examples])
53
+ return self._result(value, len(examples))
54
+
55
+ def per_example(self, example: Example) -> float | None:
56
+ ok = example.parsed is not None and example.error != ErrorCode.SCHEMA_VIOLATION
57
+ return 1.0 if ok else 0.0
58
+
59
+
60
+ class _FieldMetric(Metric):
61
+ def compute(self, examples: Sequence[Example]) -> MetricValue:
62
+ scores = [s for e in examples if (s := self.per_example(e)) is not None]
63
+ return self._result(_mean(scores), len(scores))
64
+
65
+
66
+ class FieldCompleteness(_FieldMetric):
67
+ """Average share of expected fields that are present in the output."""
68
+
69
+ name = "field_completeness"
70
+
71
+ def per_example(self, example: Example) -> float | None:
72
+ expected = _expected_fields(example)
73
+ if not expected:
74
+ return None
75
+ predicted = _predicted_fields(example)
76
+ return sum(key in predicted for key in expected) / len(expected)
77
+
78
+
79
+ class FieldAccuracy(_FieldMetric):
80
+ """Average share of expected fields whose value matches exactly."""
81
+
82
+ name = "field_accuracy"
83
+
84
+ def per_example(self, example: Example) -> float | None:
85
+ expected = _expected_fields(example)
86
+ if not expected:
87
+ return None
88
+ predicted = _predicted_fields(example)
89
+ return sum(predicted.get(key) == value for key, value in expected.items()) / len(expected)
90
+
91
+
92
+ class FieldF1(_FieldMetric):
93
+ """Average F1 between the output's and the expected object's (field, value) pairs."""
94
+
95
+ name = "field_f1"
96
+
97
+ def per_example(self, example: Example) -> float | None:
98
+ expected = _expected_fields(example)
99
+ if expected is None:
100
+ return None
101
+ predicted = _predicted_fields(example)
102
+ expected_pairs = set(expected.items())
103
+ predicted_pairs = set(predicted.items())
104
+ hits = len(expected_pairs & predicted_pairs)
105
+ if not hits:
106
+ return 1.0 if not expected_pairs and not predicted_pairs else 0.0
107
+ precision = hits / len(predicted_pairs)
108
+ recall = hits / len(expected_pairs)
109
+ return 2 * precision * recall / (precision + recall)
strop/metrics/suite.py ADDED
@@ -0,0 +1,109 @@
1
+ """Evaluating the full set of metrics a project asks for."""
2
+
3
+ from collections.abc import Mapping, Sequence
4
+ from dataclasses import dataclass
5
+ from typing import Any
6
+
7
+ from strop.config.metrics import Bound, MetricsConfig
8
+ from strop.metrics.registry import MetricContext, build_metric
9
+ from strop.metrics.types import Direction, Example, Metric, MetricValue
10
+
11
+ ALWAYS_REPORTED = "valid_output_rate"
12
+
13
+
14
+ @dataclass(frozen=True)
15
+ class MetricReport:
16
+ """Every metric value computed for one set of examples."""
17
+
18
+ primary: str
19
+ values: dict[str, MetricValue]
20
+
21
+ @property
22
+ def primary_value(self) -> MetricValue:
23
+ return self.values[self.primary]
24
+
25
+ def to_dict(self) -> dict[str, Any]:
26
+ return {
27
+ "primary": self.primary,
28
+ "values": {
29
+ name: {
30
+ "value": m.value,
31
+ "samples": m.samples,
32
+ "subjective": m.subjective,
33
+ "details": m.details,
34
+ }
35
+ for name, m in self.values.items()
36
+ },
37
+ }
38
+
39
+ @classmethod
40
+ def from_dict(cls, data: Mapping[str, Any]) -> "MetricReport":
41
+ values = {
42
+ name: MetricValue(
43
+ name, item["value"], item["samples"], item["subjective"], item["details"]
44
+ )
45
+ for name, item in data["values"].items()
46
+ }
47
+ return cls(data["primary"], values)
48
+
49
+ def violations(self, bounds: Mapping[str, Bound]) -> list[str]:
50
+ """Constraint violations as readable sentences; empty when all constraints hold."""
51
+ problems = []
52
+ for name, bound in bounds.items():
53
+ metric = self.values.get(name)
54
+ if metric is None or metric.value is None:
55
+ problems.append(f"{name} could not be computed")
56
+ elif bound.min is not None and metric.value < bound.min:
57
+ problems.append(f"{name} is {metric.value:.4f}, below the minimum {bound.min}")
58
+ elif bound.max is not None and metric.value > bound.max:
59
+ problems.append(f"{name} is {metric.value:.4f}, above the maximum {bound.max}")
60
+ return problems
61
+
62
+
63
+ class MetricSuite:
64
+ """The primary metric, secondary metrics, rule checks, and constraint metrics together."""
65
+
66
+ def __init__(self, primary: str, metrics: Mapping[str, Metric]) -> None:
67
+ self.primary = primary
68
+ self._metrics = dict(metrics)
69
+
70
+ @classmethod
71
+ def from_config(
72
+ cls,
73
+ config: MetricsConfig,
74
+ context: MetricContext,
75
+ extra: Mapping[str, Metric] | None = None,
76
+ ) -> "MetricSuite":
77
+ extra = extra or {}
78
+ names = [
79
+ config.primary,
80
+ *config.secondary,
81
+ *config.constraints,
82
+ *(f"rule:{name}" for name in config.rules),
83
+ *extra,
84
+ ALWAYS_REPORTED,
85
+ ]
86
+ metrics = {
87
+ name: extra[name] if name in extra else build_metric(name, context)
88
+ for name in dict.fromkeys(names)
89
+ }
90
+ return cls(config.primary, metrics)
91
+
92
+ @property
93
+ def names(self) -> list[str]:
94
+ return list(self._metrics)
95
+
96
+ def metric(self, name: str) -> Metric:
97
+ return self._metrics[name]
98
+
99
+ def direction(self, name: str) -> Direction:
100
+ return self._metrics[name].direction
101
+
102
+ async def prepare(self, examples: Sequence[Example]) -> None:
103
+ """Run every metric's batched or asynchronous setup before scoring."""
104
+ for metric in self._metrics.values():
105
+ await metric.prepare(examples)
106
+
107
+ def evaluate(self, examples: Sequence[Example]) -> MetricReport:
108
+ values = {name: metric.compute(examples) for name, metric in self._metrics.items()}
109
+ return MetricReport(self.primary, values)
strop/metrics/text.py ADDED
@@ -0,0 +1,55 @@
1
+ """Metrics comparing text or numeric outputs with the expected value."""
2
+
3
+ import difflib
4
+ from collections.abc import Sequence
5
+
6
+ from strop.metrics.types import Example, Metric, MetricValue
7
+
8
+ DEFAULT_TOLERANCE = 1e-6
9
+
10
+
11
+ def _to_float(text: str) -> float | None:
12
+ try:
13
+ return float(text.strip().replace(",", ""))
14
+ except ValueError:
15
+ return None
16
+
17
+
18
+ class _MeanScore(Metric):
19
+ def compute(self, examples: Sequence[Example]) -> MetricValue:
20
+ scores = [s for e in examples if (s := self.per_example(e)) is not None]
21
+ return self._result(sum(scores) / len(scores) if scores else None, len(scores))
22
+
23
+
24
+ class Similarity(_MeanScore):
25
+ """Average character-level similarity (0 to 1) between output and expected text."""
26
+
27
+ name = "similarity"
28
+
29
+ def per_example(self, example: Example) -> float | None:
30
+ if example.expected is None:
31
+ return None
32
+ if example.value is None:
33
+ return 0.0
34
+ return difflib.SequenceMatcher(None, example.value, example.expected).ratio()
35
+
36
+
37
+ class NumericClose(_MeanScore):
38
+ """Share of outputs whose number is within an absolute tolerance of the expected number."""
39
+
40
+ def __init__(self, tolerance: float = DEFAULT_TOLERANCE) -> None:
41
+ self._tolerance = tolerance
42
+ self.name = (
43
+ "numeric_close" if tolerance == DEFAULT_TOLERANCE else f"numeric_close:{tolerance:g}"
44
+ )
45
+
46
+ def per_example(self, example: Example) -> float | None:
47
+ if example.expected is None:
48
+ return None
49
+ expected = _to_float(example.expected)
50
+ predicted = _to_float(example.value) if example.value is not None else None
51
+ if expected is None:
52
+ return None
53
+ if predicted is None:
54
+ return 0.0
55
+ return 1.0 if abs(predicted - expected) <= self._tolerance else 0.0
strop/metrics/types.py ADDED
@@ -0,0 +1,106 @@
1
+ """Core metric types."""
2
+
3
+ import json
4
+ from abc import ABC, abstractmethod
5
+ from collections.abc import Mapping, Sequence
6
+ from dataclasses import dataclass, field
7
+ from enum import StrEnum
8
+ from typing import Any
9
+
10
+
11
+ class Direction(StrEnum):
12
+ MAXIMIZE = "maximize"
13
+ MINIMIZE = "minimize"
14
+
15
+
16
+ class ErrorCode(StrEnum):
17
+ """Deterministic failure categories attached to individual examples."""
18
+
19
+ PROVIDER_ERROR = "provider_error"
20
+ EMPTY_OUTPUT = "empty_output"
21
+ TRUNCATED = "truncated"
22
+ INVALID_FORMAT = "invalid_format"
23
+ SCHEMA_VIOLATION = "schema_violation"
24
+ WRONG_LABEL = "wrong_label"
25
+ WRONG_VALUE = "wrong_value"
26
+ MISSING_FIELD = "missing_field"
27
+ RULE_VIOLATION = "rule_violation"
28
+ LOW_SCORE = "low_score"
29
+ METRIC_ERROR = "metric_error"
30
+
31
+
32
+ @dataclass(frozen=True, slots=True)
33
+ class Example:
34
+ """One model output paired with its row, ready to be scored."""
35
+
36
+ row_id: str
37
+ inputs: Mapping[str, str]
38
+ expected: str | None
39
+ raw_output: str
40
+ value: str | None = None
41
+ parsed: Any = None
42
+ error: str | None = None
43
+
44
+ @property
45
+ def valid(self) -> bool:
46
+ """True if the output was produced and parsed under the output contract."""
47
+ return self.value is not None
48
+
49
+ @property
50
+ def correct(self) -> bool:
51
+ return self.value is not None and self.value == self.expected
52
+
53
+ @property
54
+ def expected_json(self) -> Any:
55
+ """The expected output parsed as JSON, or None when absent or not JSON."""
56
+ if self.expected is None:
57
+ return None
58
+ try:
59
+ return json.loads(self.expected)
60
+ except ValueError:
61
+ return None
62
+
63
+
64
+ @dataclass(frozen=True)
65
+ class MetricValue:
66
+ """A computed metric. ``value`` is None when it cannot be computed (no scorable examples)."""
67
+
68
+ name: str
69
+ value: float | None
70
+ samples: int
71
+ subjective: bool = False
72
+ details: dict[str, Any] = field(default_factory=dict)
73
+
74
+
75
+ class Metric(ABC):
76
+ """A named score over a set of examples."""
77
+
78
+ name: str
79
+ direction: Direction = Direction.MAXIMIZE
80
+ subjective: bool = False
81
+ needs_expected: bool = True
82
+
83
+ @abstractmethod
84
+ def compute(self, examples: Sequence[Example]) -> MetricValue:
85
+ """Aggregate score over ``examples``."""
86
+
87
+ async def prepare(self, examples: Sequence[Example]) -> None:
88
+ """Do any batched or asynchronous work (model calls, subprocesses) before scoring."""
89
+ return None
90
+
91
+ def per_example(self, example: Example) -> float | None:
92
+ """A per-example score used for paired comparison; None if not applicable.
93
+
94
+ The default is correctness (1 or 0), or None when there is no expected value.
95
+ Metrics that score on another scale override it.
96
+ """
97
+ if example.expected is None:
98
+ return None
99
+ return 1.0 if example.correct else 0.0
100
+
101
+ def feedback(self, example: Example) -> str | None:
102
+ """A short explanation of this example's score, shown to the optimizer."""
103
+ return None
104
+
105
+ def _result(self, value: float | None, samples: int, **details: Any) -> MetricValue:
106
+ return MetricValue(self.name, value, samples, self.subjective, dict(details))
@@ -0,0 +1,25 @@
1
+ """Prompt optimization: proposing revisions and deciding whether to keep them."""
2
+
3
+ from strop.optimization.acceptance import AcceptanceRules, Decision, compare
4
+ from strop.optimization.generator import CandidateGenerator, OptimizerOutput, apply_output
5
+ from strop.optimization.types import (
6
+ Candidate,
7
+ HistoryEntry,
8
+ MetricSummary,
9
+ OptimizerView,
10
+ RejectedProposal,
11
+ )
12
+
13
+ __all__ = [
14
+ "AcceptanceRules",
15
+ "Candidate",
16
+ "CandidateGenerator",
17
+ "Decision",
18
+ "HistoryEntry",
19
+ "MetricSummary",
20
+ "OptimizerOutput",
21
+ "OptimizerView",
22
+ "RejectedProposal",
23
+ "apply_output",
24
+ "compare",
25
+ ]