strop-cli 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- strop/__init__.py +5 -0
- strop/__main__.py +5 -0
- strop/analysis/__init__.py +17 -0
- strop/analysis/classify.py +52 -0
- strop/analysis/evidence.py +52 -0
- strop/analysis/facts.py +74 -0
- strop/analysis/patterns.py +90 -0
- strop/cli/__init__.py +5 -0
- strop/cli/analyze.py +93 -0
- strop/cli/app.py +82 -0
- strop/cli/data.py +111 -0
- strop/cli/evaluate.py +145 -0
- strop/cli/export.py +74 -0
- strop/cli/final.py +106 -0
- strop/cli/init.py +131 -0
- strop/cli/optimize.py +156 -0
- strop/cli/options.py +20 -0
- strop/cli/progress.py +18 -0
- strop/cli/prompts.py +110 -0
- strop/cli/status.py +68 -0
- strop/cli/versions.py +123 -0
- strop/config/__init__.py +16 -0
- strop/config/base.py +9 -0
- strop/config/constraints.py +33 -0
- strop/config/dataset.py +63 -0
- strop/config/enums.py +52 -0
- strop/config/io.py +52 -0
- strop/config/limits.py +19 -0
- strop/config/llm.py +42 -0
- strop/config/metrics.py +86 -0
- strop/config/optimization.py +23 -0
- strop/config/output.py +31 -0
- strop/config/overrides.py +22 -0
- strop/config/root.py +36 -0
- strop/config/rules.py +67 -0
- strop/config/split.py +23 -0
- strop/data/__init__.py +58 -0
- strop/data/loaders.py +85 -0
- strop/data/models.py +44 -0
- strop/data/rows.py +99 -0
- strop/data/splitting.py +103 -0
- strop/env.py +44 -0
- strop/errors.py +59 -0
- strop/evaluation/__init__.py +1 -0
- strop/evaluation/budget.py +67 -0
- strop/evaluation/budgeted.py +23 -0
- strop/evaluation/examples.py +49 -0
- strop/evaluation/models.py +105 -0
- strop/evaluation/parsing.py +96 -0
- strop/evaluation/runner.py +107 -0
- strop/formatting.py +14 -0
- strop/llm/__init__.py +7 -0
- strop/llm/caching.py +28 -0
- strop/llm/client.py +15 -0
- strop/llm/factory.py +14 -0
- strop/llm/litellm_client.py +90 -0
- strop/llm/mock.py +43 -0
- strop/llm/retry.py +44 -0
- strop/llm/structured.py +52 -0
- strop/llm/types.py +59 -0
- strop/log.py +49 -0
- strop/metrics/__init__.py +1 -0
- strop/metrics/basic.py +30 -0
- strop/metrics/classification.py +164 -0
- strop/metrics/judge.py +130 -0
- strop/metrics/python_metric.py +101 -0
- strop/metrics/python_runner.py +65 -0
- strop/metrics/registry.py +115 -0
- strop/metrics/rules.py +55 -0
- strop/metrics/structured.py +109 -0
- strop/metrics/suite.py +109 -0
- strop/metrics/text.py +55 -0
- strop/metrics/types.py +106 -0
- strop/optimization/__init__.py +25 -0
- strop/optimization/acceptance.py +170 -0
- strop/optimization/generator.py +92 -0
- strop/optimization/meta_prompt.py +101 -0
- strop/optimization/types.py +69 -0
- strop/prompts/__init__.py +18 -0
- strop/prompts/diff.py +139 -0
- strop/prompts/files.py +92 -0
- strop/prompts/models.py +51 -0
- strop/prompts/render.py +16 -0
- strop/prompts/template.py +45 -0
- strop/prompts/validation.py +31 -0
- strop/prompts/versions.py +34 -0
- strop/py.typed +0 -0
- strop/reporting/__init__.py +6 -0
- strop/reporting/console.py +20 -0
- strop/reporting/diff_view.py +102 -0
- strop/reporting/evaluation_view.py +141 -0
- strop/reporting/final_view.py +109 -0
- strop/reporting/prompt_export.py +58 -0
- strop/reporting/report_markdown.py +157 -0
- strop/reporting/run_view.py +118 -0
- strop/services/__init__.py +1 -0
- strop/services/analysis.py +124 -0
- strop/services/clean.py +38 -0
- strop/services/compare.py +60 -0
- strop/services/context.py +70 -0
- strop/services/data.py +65 -0
- strop/services/evaluate.py +105 -0
- strop/services/evaluator.py +169 -0
- strop/services/export.py +90 -0
- strop/services/final_test.py +69 -0
- strop/services/metrics.py +89 -0
- strop/services/optimize.py +187 -0
- strop/services/partitions.py +46 -0
- strop/services/project.py +55 -0
- strop/services/prompts.py +76 -0
- strop/services/report.py +194 -0
- strop/services/review.py +43 -0
- strop/services/scaffold.py +79 -0
- strop/services/turn.py +234 -0
- strop/storage/__init__.py +6 -0
- strop/storage/cache.py +45 -0
- strop/storage/database.py +45 -0
- strop/storage/datasets.py +129 -0
- strop/storage/evaluations.py +160 -0
- strop/storage/migrations/001_initial.sql +5 -0
- strop/storage/migrations/002_datasets.sql +27 -0
- strop/storage/migrations/003_prompt_versions.sql +22 -0
- strop/storage/migrations/004_llm_cache.sql +6 -0
- strop/storage/migrations/005_evaluations.sql +37 -0
- strop/storage/migrations/006_runs.sql +52 -0
- strop/storage/migrations/007_example_feedback.sql +2 -0
- strop/storage/migrations/__init__.py +5 -0
- strop/storage/migrator.py +91 -0
- strop/storage/paths.py +30 -0
- strop/storage/prompts.py +96 -0
- strop/storage/run_models.py +75 -0
- strop/storage/runs.py +263 -0
- strop_cli-0.1.0.dist-info/METADATA +242 -0
- strop_cli-0.1.0.dist-info/RECORD +137 -0
- strop_cli-0.1.0.dist-info/WHEEL +4 -0
- strop_cli-0.1.0.dist-info/entry_points.txt +2 -0
- strop_cli-0.1.0.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
"""Runs a user's metric file on a batch of examples.
|
|
2
|
+
|
|
3
|
+
This file is executed as a script in a separate Python process by ``python_metric``. It
|
|
4
|
+
imports nothing from Strop. The user's ``score(output, row)`` function may return a number
|
|
5
|
+
between 0 and 1, a bool, or a dict ``{"score": number, "feedback": "text"}``.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
import importlib.util
|
|
9
|
+
import json
|
|
10
|
+
import sys
|
|
11
|
+
from pathlib import Path
|
|
12
|
+
from typing import Any
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def _normalize(value: Any) -> dict[str, Any]:
|
|
16
|
+
if isinstance(value, bool):
|
|
17
|
+
return {"score": 1.0 if value else 0.0}
|
|
18
|
+
if isinstance(value, int | float):
|
|
19
|
+
return _checked(float(value), None)
|
|
20
|
+
if isinstance(value, dict) and isinstance(value.get("score"), int | float):
|
|
21
|
+
feedback = value.get("feedback")
|
|
22
|
+
return _checked(float(value["score"]), str(feedback) if feedback else None)
|
|
23
|
+
return {"error": "score() must return a number or a dict with a 'score' key"}
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def _checked(score: float, feedback: str | None) -> dict[str, Any]:
|
|
27
|
+
if not 0.0 <= score <= 1.0:
|
|
28
|
+
return {"error": f"score {score} is outside the range 0 to 1"}
|
|
29
|
+
return {"score": score, "feedback": feedback}
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def _load(path: str) -> Any:
|
|
33
|
+
spec = importlib.util.spec_from_file_location("strop_user_metric", path)
|
|
34
|
+
if spec is None or spec.loader is None:
|
|
35
|
+
raise ImportError(f"cannot load {path}")
|
|
36
|
+
module = importlib.util.module_from_spec(spec)
|
|
37
|
+
sys.path.insert(0, str(Path(path).resolve().parent))
|
|
38
|
+
spec.loader.exec_module(module)
|
|
39
|
+
if not callable(getattr(module, "score", None)):
|
|
40
|
+
raise AttributeError("the metric file must define a function score(output, row)")
|
|
41
|
+
return module.score
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def main(path: str) -> int:
|
|
45
|
+
real_stdout = sys.stdout
|
|
46
|
+
sys.stdout = sys.stderr # user print() calls must not corrupt the JSON result
|
|
47
|
+
try:
|
|
48
|
+
score = _load(path)
|
|
49
|
+
except Exception as exc:
|
|
50
|
+
sys.stdout = real_stdout
|
|
51
|
+
json.dump({"fatal": f"{type(exc).__name__}: {exc}"}, sys.stdout)
|
|
52
|
+
return 3
|
|
53
|
+
results = []
|
|
54
|
+
for item in json.load(sys.stdin):
|
|
55
|
+
try:
|
|
56
|
+
results.append(_normalize(score(item["output"], item["row"])))
|
|
57
|
+
except Exception as exc:
|
|
58
|
+
results.append({"error": f"{type(exc).__name__}: {exc}"})
|
|
59
|
+
sys.stdout = real_stdout
|
|
60
|
+
json.dump({"results": results}, sys.stdout)
|
|
61
|
+
return 0
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
if __name__ == "__main__":
|
|
65
|
+
sys.exit(main(sys.argv[1]))
|
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
"""Resolving metric names from strop.yaml to metric objects."""
|
|
2
|
+
|
|
3
|
+
from collections.abc import Callable, Mapping
|
|
4
|
+
from dataclasses import dataclass, field
|
|
5
|
+
|
|
6
|
+
from strop.config.metrics import RuleSpec
|
|
7
|
+
from strop.errors import ConfigError
|
|
8
|
+
from strop.metrics.basic import ExactMatch, ValidOutputRate
|
|
9
|
+
from strop.metrics.classification import Accuracy, Average, Averaged, ClassScore, Kind
|
|
10
|
+
from strop.metrics.rules import RuleMetric
|
|
11
|
+
from strop.metrics.structured import (
|
|
12
|
+
FieldAccuracy,
|
|
13
|
+
FieldCompleteness,
|
|
14
|
+
FieldF1,
|
|
15
|
+
JsonValid,
|
|
16
|
+
SchemaValid,
|
|
17
|
+
)
|
|
18
|
+
from strop.metrics.text import NumericClose, Similarity
|
|
19
|
+
from strop.metrics.types import Metric
|
|
20
|
+
|
|
21
|
+
_AVERAGES: tuple[Average, ...] = ("macro", "micro", "weighted")
|
|
22
|
+
_KINDS: tuple[Kind, ...] = ("precision", "recall", "f1")
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
@dataclass(frozen=True)
|
|
26
|
+
class MetricContext:
|
|
27
|
+
"""What metric constructors need to know about the task."""
|
|
28
|
+
|
|
29
|
+
labels: tuple[str, ...] = ()
|
|
30
|
+
rules: Mapping[str, RuleSpec] = field(default_factory=dict)
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
Factory = Callable[[MetricContext], Metric]
|
|
34
|
+
ParamFactory = Callable[[str, MetricContext], Metric]
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def _averaged_factory(average: Average, kind: Kind) -> Factory:
|
|
38
|
+
return lambda ctx: Averaged(average, kind, ctx.labels)
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def _simple_factories() -> dict[str, Factory]:
|
|
42
|
+
factories: dict[str, Factory] = {
|
|
43
|
+
"accuracy": lambda ctx: Accuracy(ctx.labels),
|
|
44
|
+
"exact_match": lambda ctx: ExactMatch(),
|
|
45
|
+
"valid_output_rate": lambda ctx: ValidOutputRate(),
|
|
46
|
+
"json_valid": lambda ctx: JsonValid(),
|
|
47
|
+
"schema_valid": lambda ctx: SchemaValid(),
|
|
48
|
+
"field_completeness": lambda ctx: FieldCompleteness(),
|
|
49
|
+
"field_accuracy": lambda ctx: FieldAccuracy(),
|
|
50
|
+
"field_f1": lambda ctx: FieldF1(),
|
|
51
|
+
"similarity": lambda ctx: Similarity(),
|
|
52
|
+
"numeric_close": lambda ctx: NumericClose(),
|
|
53
|
+
}
|
|
54
|
+
for average in _AVERAGES:
|
|
55
|
+
for kind in _KINDS:
|
|
56
|
+
factories[f"{average}_{kind}"] = _averaged_factory(average, kind)
|
|
57
|
+
return factories
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def _class_factory(kind: Kind) -> ParamFactory:
|
|
61
|
+
def build(label: str, ctx: MetricContext) -> Metric:
|
|
62
|
+
if ctx.labels and label not in ctx.labels:
|
|
63
|
+
raise ConfigError(
|
|
64
|
+
f"metric '{kind}:{label}' names an unknown label; labels: {', '.join(ctx.labels)}"
|
|
65
|
+
)
|
|
66
|
+
return ClassScore(kind, label, ctx.labels)
|
|
67
|
+
|
|
68
|
+
return build
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def _rule_factory(name: str, ctx: MetricContext) -> Metric:
|
|
72
|
+
if name not in ctx.rules:
|
|
73
|
+
defined = ", ".join(ctx.rules) or "none"
|
|
74
|
+
raise ConfigError(f"rule '{name}' is not defined under metrics.rules; defined: {defined}")
|
|
75
|
+
return RuleMetric(name, ctx.rules[name])
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def _numeric_close_factory(tolerance: str, ctx: MetricContext) -> Metric:
|
|
79
|
+
try:
|
|
80
|
+
return NumericClose(float(tolerance))
|
|
81
|
+
except ValueError:
|
|
82
|
+
raise ConfigError(f"metric 'numeric_close:{tolerance}' needs a numeric tolerance") from None
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
_SIMPLE = _simple_factories()
|
|
86
|
+
_PARAMETERIZED: dict[str, ParamFactory] = {
|
|
87
|
+
**{kind: _class_factory(kind) for kind in _KINDS},
|
|
88
|
+
"rule": _rule_factory,
|
|
89
|
+
"numeric_close": _numeric_close_factory,
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def known_metric_names() -> list[str]:
|
|
94
|
+
"""Built-in metric names, plus the parameterized forms."""
|
|
95
|
+
return sorted(_SIMPLE) + [f"{prefix}:<label>" for prefix in sorted(_PARAMETERIZED)]
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def register_parameterized(prefix: str, factory: ParamFactory) -> None:
|
|
99
|
+
"""Add a metric family addressed as ``<prefix>:<argument>``."""
|
|
100
|
+
_PARAMETERIZED[prefix] = factory
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
def register(name: str, factory: Factory) -> None:
|
|
104
|
+
"""Add a built-in metric addressed by ``name``."""
|
|
105
|
+
_SIMPLE[name] = factory
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
def build_metric(name: str, context: MetricContext) -> Metric:
|
|
109
|
+
"""Create the metric called ``name`` or raise ``ConfigError`` listing the valid names."""
|
|
110
|
+
if name in _SIMPLE:
|
|
111
|
+
return _SIMPLE[name](context)
|
|
112
|
+
prefix, separator, argument = name.partition(":")
|
|
113
|
+
if separator and argument and prefix in _PARAMETERIZED:
|
|
114
|
+
return _PARAMETERIZED[prefix](argument, context)
|
|
115
|
+
raise ConfigError(f"unknown metric '{name}'; available: {', '.join(known_metric_names())}")
|
strop/metrics/rules.py
ADDED
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
"""Rule checks: deterministic pass/fail tests on each output, reported as pass rates."""
|
|
2
|
+
|
|
3
|
+
import re
|
|
4
|
+
from collections.abc import Callable, Sequence
|
|
5
|
+
|
|
6
|
+
from strop.config.enums import RuleType
|
|
7
|
+
from strop.config.metrics import RuleSpec
|
|
8
|
+
from strop.metrics.types import ErrorCode, Example, Metric, MetricValue
|
|
9
|
+
|
|
10
|
+
Check = Callable[[str, RuleSpec], bool]
|
|
11
|
+
|
|
12
|
+
_CHECKS: dict[RuleType, Check] = {
|
|
13
|
+
RuleType.MAX_CHARS: lambda text, spec: len(text) <= int(str(spec.value)),
|
|
14
|
+
RuleType.MIN_CHARS: lambda text, spec: len(text) >= int(str(spec.value)),
|
|
15
|
+
RuleType.MAX_WORDS: lambda text, spec: len(text.split()) <= int(str(spec.value)),
|
|
16
|
+
RuleType.MIN_WORDS: lambda text, spec: len(text.split()) >= int(str(spec.value)),
|
|
17
|
+
RuleType.MATCHES: lambda text, spec: re.search(str(spec.value), text) is not None,
|
|
18
|
+
RuleType.NOT_MATCHES: lambda text, spec: re.search(str(spec.value), text) is None,
|
|
19
|
+
RuleType.CONTAINS_ALL: lambda text, spec: all(
|
|
20
|
+
term.casefold() in text.casefold() for term in _terms(spec)
|
|
21
|
+
),
|
|
22
|
+
RuleType.CONTAINS_NONE: lambda text, spec: (
|
|
23
|
+
not any(term.casefold() in text.casefold() for term in _terms(spec))
|
|
24
|
+
),
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def _terms(spec: RuleSpec) -> list[str]:
|
|
29
|
+
return spec.value if isinstance(spec.value, list) else [str(spec.value)]
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
class RuleMetric(Metric):
|
|
33
|
+
"""Pass rate of one rule, named ``rule:<name>``.
|
|
34
|
+
|
|
35
|
+
Outputs lost to provider failures are not scored; every other output, including an
|
|
36
|
+
empty one, is checked as written.
|
|
37
|
+
"""
|
|
38
|
+
|
|
39
|
+
needs_expected = False
|
|
40
|
+
|
|
41
|
+
def __init__(self, rule_name: str, spec: RuleSpec) -> None:
|
|
42
|
+
self.name = f"rule:{rule_name}"
|
|
43
|
+
self._spec = spec
|
|
44
|
+
|
|
45
|
+
def compute(self, examples: Sequence[Example]) -> MetricValue:
|
|
46
|
+
scores = [s for e in examples if (s := self.per_example(e)) is not None]
|
|
47
|
+
return self._result(sum(scores) / len(scores) if scores else None, len(scores))
|
|
48
|
+
|
|
49
|
+
def per_example(self, example: Example) -> float | None:
|
|
50
|
+
if example.error == ErrorCode.PROVIDER_ERROR:
|
|
51
|
+
return None
|
|
52
|
+
text = example.raw_output.strip()
|
|
53
|
+
if not text:
|
|
54
|
+
return 0.0
|
|
55
|
+
return 1.0 if _CHECKS[self._spec.type](text, self._spec) else 0.0
|
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
"""Metrics for JSON outputs: validity and field-level comparison against the expected object."""
|
|
2
|
+
|
|
3
|
+
import json
|
|
4
|
+
from collections.abc import Sequence
|
|
5
|
+
from typing import Any
|
|
6
|
+
|
|
7
|
+
from strop.metrics.types import ErrorCode, Example, Metric, MetricValue
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def _canonical(value: Any) -> str:
|
|
11
|
+
return json.dumps(value, sort_keys=True, ensure_ascii=False)
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def _expected_fields(example: Example) -> dict[str, str] | None:
|
|
15
|
+
expected = example.expected_json
|
|
16
|
+
if not isinstance(expected, dict):
|
|
17
|
+
return None
|
|
18
|
+
return {key: _canonical(value) for key, value in expected.items()}
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def _predicted_fields(example: Example) -> dict[str, str]:
|
|
22
|
+
if not isinstance(example.parsed, dict):
|
|
23
|
+
return {}
|
|
24
|
+
return {key: _canonical(value) for key, value in example.parsed.items()}
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def _mean(scores: list[float]) -> float | None:
|
|
28
|
+
return sum(scores) / len(scores) if scores else None
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
class JsonValid(Metric):
|
|
32
|
+
"""Share of outputs that parse as JSON, before any schema check."""
|
|
33
|
+
|
|
34
|
+
name = "json_valid"
|
|
35
|
+
needs_expected = False
|
|
36
|
+
|
|
37
|
+
def compute(self, examples: Sequence[Example]) -> MetricValue:
|
|
38
|
+
value = _mean([self.per_example(e) or 0.0 for e in examples])
|
|
39
|
+
return self._result(value, len(examples))
|
|
40
|
+
|
|
41
|
+
def per_example(self, example: Example) -> float | None:
|
|
42
|
+
return 1.0 if example.parsed is not None else 0.0
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
class SchemaValid(Metric):
|
|
46
|
+
"""Share of outputs that parse as JSON and satisfy the configured schema."""
|
|
47
|
+
|
|
48
|
+
name = "schema_valid"
|
|
49
|
+
needs_expected = False
|
|
50
|
+
|
|
51
|
+
def compute(self, examples: Sequence[Example]) -> MetricValue:
|
|
52
|
+
value = _mean([self.per_example(e) or 0.0 for e in examples])
|
|
53
|
+
return self._result(value, len(examples))
|
|
54
|
+
|
|
55
|
+
def per_example(self, example: Example) -> float | None:
|
|
56
|
+
ok = example.parsed is not None and example.error != ErrorCode.SCHEMA_VIOLATION
|
|
57
|
+
return 1.0 if ok else 0.0
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
class _FieldMetric(Metric):
|
|
61
|
+
def compute(self, examples: Sequence[Example]) -> MetricValue:
|
|
62
|
+
scores = [s for e in examples if (s := self.per_example(e)) is not None]
|
|
63
|
+
return self._result(_mean(scores), len(scores))
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
class FieldCompleteness(_FieldMetric):
|
|
67
|
+
"""Average share of expected fields that are present in the output."""
|
|
68
|
+
|
|
69
|
+
name = "field_completeness"
|
|
70
|
+
|
|
71
|
+
def per_example(self, example: Example) -> float | None:
|
|
72
|
+
expected = _expected_fields(example)
|
|
73
|
+
if not expected:
|
|
74
|
+
return None
|
|
75
|
+
predicted = _predicted_fields(example)
|
|
76
|
+
return sum(key in predicted for key in expected) / len(expected)
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
class FieldAccuracy(_FieldMetric):
|
|
80
|
+
"""Average share of expected fields whose value matches exactly."""
|
|
81
|
+
|
|
82
|
+
name = "field_accuracy"
|
|
83
|
+
|
|
84
|
+
def per_example(self, example: Example) -> float | None:
|
|
85
|
+
expected = _expected_fields(example)
|
|
86
|
+
if not expected:
|
|
87
|
+
return None
|
|
88
|
+
predicted = _predicted_fields(example)
|
|
89
|
+
return sum(predicted.get(key) == value for key, value in expected.items()) / len(expected)
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
class FieldF1(_FieldMetric):
|
|
93
|
+
"""Average F1 between the output's and the expected object's (field, value) pairs."""
|
|
94
|
+
|
|
95
|
+
name = "field_f1"
|
|
96
|
+
|
|
97
|
+
def per_example(self, example: Example) -> float | None:
|
|
98
|
+
expected = _expected_fields(example)
|
|
99
|
+
if expected is None:
|
|
100
|
+
return None
|
|
101
|
+
predicted = _predicted_fields(example)
|
|
102
|
+
expected_pairs = set(expected.items())
|
|
103
|
+
predicted_pairs = set(predicted.items())
|
|
104
|
+
hits = len(expected_pairs & predicted_pairs)
|
|
105
|
+
if not hits:
|
|
106
|
+
return 1.0 if not expected_pairs and not predicted_pairs else 0.0
|
|
107
|
+
precision = hits / len(predicted_pairs)
|
|
108
|
+
recall = hits / len(expected_pairs)
|
|
109
|
+
return 2 * precision * recall / (precision + recall)
|
strop/metrics/suite.py
ADDED
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
"""Evaluating the full set of metrics a project asks for."""
|
|
2
|
+
|
|
3
|
+
from collections.abc import Mapping, Sequence
|
|
4
|
+
from dataclasses import dataclass
|
|
5
|
+
from typing import Any
|
|
6
|
+
|
|
7
|
+
from strop.config.metrics import Bound, MetricsConfig
|
|
8
|
+
from strop.metrics.registry import MetricContext, build_metric
|
|
9
|
+
from strop.metrics.types import Direction, Example, Metric, MetricValue
|
|
10
|
+
|
|
11
|
+
ALWAYS_REPORTED = "valid_output_rate"
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
@dataclass(frozen=True)
|
|
15
|
+
class MetricReport:
|
|
16
|
+
"""Every metric value computed for one set of examples."""
|
|
17
|
+
|
|
18
|
+
primary: str
|
|
19
|
+
values: dict[str, MetricValue]
|
|
20
|
+
|
|
21
|
+
@property
|
|
22
|
+
def primary_value(self) -> MetricValue:
|
|
23
|
+
return self.values[self.primary]
|
|
24
|
+
|
|
25
|
+
def to_dict(self) -> dict[str, Any]:
|
|
26
|
+
return {
|
|
27
|
+
"primary": self.primary,
|
|
28
|
+
"values": {
|
|
29
|
+
name: {
|
|
30
|
+
"value": m.value,
|
|
31
|
+
"samples": m.samples,
|
|
32
|
+
"subjective": m.subjective,
|
|
33
|
+
"details": m.details,
|
|
34
|
+
}
|
|
35
|
+
for name, m in self.values.items()
|
|
36
|
+
},
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
@classmethod
|
|
40
|
+
def from_dict(cls, data: Mapping[str, Any]) -> "MetricReport":
|
|
41
|
+
values = {
|
|
42
|
+
name: MetricValue(
|
|
43
|
+
name, item["value"], item["samples"], item["subjective"], item["details"]
|
|
44
|
+
)
|
|
45
|
+
for name, item in data["values"].items()
|
|
46
|
+
}
|
|
47
|
+
return cls(data["primary"], values)
|
|
48
|
+
|
|
49
|
+
def violations(self, bounds: Mapping[str, Bound]) -> list[str]:
|
|
50
|
+
"""Constraint violations as readable sentences; empty when all constraints hold."""
|
|
51
|
+
problems = []
|
|
52
|
+
for name, bound in bounds.items():
|
|
53
|
+
metric = self.values.get(name)
|
|
54
|
+
if metric is None or metric.value is None:
|
|
55
|
+
problems.append(f"{name} could not be computed")
|
|
56
|
+
elif bound.min is not None and metric.value < bound.min:
|
|
57
|
+
problems.append(f"{name} is {metric.value:.4f}, below the minimum {bound.min}")
|
|
58
|
+
elif bound.max is not None and metric.value > bound.max:
|
|
59
|
+
problems.append(f"{name} is {metric.value:.4f}, above the maximum {bound.max}")
|
|
60
|
+
return problems
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
class MetricSuite:
|
|
64
|
+
"""The primary metric, secondary metrics, rule checks, and constraint metrics together."""
|
|
65
|
+
|
|
66
|
+
def __init__(self, primary: str, metrics: Mapping[str, Metric]) -> None:
|
|
67
|
+
self.primary = primary
|
|
68
|
+
self._metrics = dict(metrics)
|
|
69
|
+
|
|
70
|
+
@classmethod
|
|
71
|
+
def from_config(
|
|
72
|
+
cls,
|
|
73
|
+
config: MetricsConfig,
|
|
74
|
+
context: MetricContext,
|
|
75
|
+
extra: Mapping[str, Metric] | None = None,
|
|
76
|
+
) -> "MetricSuite":
|
|
77
|
+
extra = extra or {}
|
|
78
|
+
names = [
|
|
79
|
+
config.primary,
|
|
80
|
+
*config.secondary,
|
|
81
|
+
*config.constraints,
|
|
82
|
+
*(f"rule:{name}" for name in config.rules),
|
|
83
|
+
*extra,
|
|
84
|
+
ALWAYS_REPORTED,
|
|
85
|
+
]
|
|
86
|
+
metrics = {
|
|
87
|
+
name: extra[name] if name in extra else build_metric(name, context)
|
|
88
|
+
for name in dict.fromkeys(names)
|
|
89
|
+
}
|
|
90
|
+
return cls(config.primary, metrics)
|
|
91
|
+
|
|
92
|
+
@property
|
|
93
|
+
def names(self) -> list[str]:
|
|
94
|
+
return list(self._metrics)
|
|
95
|
+
|
|
96
|
+
def metric(self, name: str) -> Metric:
|
|
97
|
+
return self._metrics[name]
|
|
98
|
+
|
|
99
|
+
def direction(self, name: str) -> Direction:
|
|
100
|
+
return self._metrics[name].direction
|
|
101
|
+
|
|
102
|
+
async def prepare(self, examples: Sequence[Example]) -> None:
|
|
103
|
+
"""Run every metric's batched or asynchronous setup before scoring."""
|
|
104
|
+
for metric in self._metrics.values():
|
|
105
|
+
await metric.prepare(examples)
|
|
106
|
+
|
|
107
|
+
def evaluate(self, examples: Sequence[Example]) -> MetricReport:
|
|
108
|
+
values = {name: metric.compute(examples) for name, metric in self._metrics.items()}
|
|
109
|
+
return MetricReport(self.primary, values)
|
strop/metrics/text.py
ADDED
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
"""Metrics comparing text or numeric outputs with the expected value."""
|
|
2
|
+
|
|
3
|
+
import difflib
|
|
4
|
+
from collections.abc import Sequence
|
|
5
|
+
|
|
6
|
+
from strop.metrics.types import Example, Metric, MetricValue
|
|
7
|
+
|
|
8
|
+
DEFAULT_TOLERANCE = 1e-6
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def _to_float(text: str) -> float | None:
|
|
12
|
+
try:
|
|
13
|
+
return float(text.strip().replace(",", ""))
|
|
14
|
+
except ValueError:
|
|
15
|
+
return None
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
class _MeanScore(Metric):
|
|
19
|
+
def compute(self, examples: Sequence[Example]) -> MetricValue:
|
|
20
|
+
scores = [s for e in examples if (s := self.per_example(e)) is not None]
|
|
21
|
+
return self._result(sum(scores) / len(scores) if scores else None, len(scores))
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
class Similarity(_MeanScore):
|
|
25
|
+
"""Average character-level similarity (0 to 1) between output and expected text."""
|
|
26
|
+
|
|
27
|
+
name = "similarity"
|
|
28
|
+
|
|
29
|
+
def per_example(self, example: Example) -> float | None:
|
|
30
|
+
if example.expected is None:
|
|
31
|
+
return None
|
|
32
|
+
if example.value is None:
|
|
33
|
+
return 0.0
|
|
34
|
+
return difflib.SequenceMatcher(None, example.value, example.expected).ratio()
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
class NumericClose(_MeanScore):
|
|
38
|
+
"""Share of outputs whose number is within an absolute tolerance of the expected number."""
|
|
39
|
+
|
|
40
|
+
def __init__(self, tolerance: float = DEFAULT_TOLERANCE) -> None:
|
|
41
|
+
self._tolerance = tolerance
|
|
42
|
+
self.name = (
|
|
43
|
+
"numeric_close" if tolerance == DEFAULT_TOLERANCE else f"numeric_close:{tolerance:g}"
|
|
44
|
+
)
|
|
45
|
+
|
|
46
|
+
def per_example(self, example: Example) -> float | None:
|
|
47
|
+
if example.expected is None:
|
|
48
|
+
return None
|
|
49
|
+
expected = _to_float(example.expected)
|
|
50
|
+
predicted = _to_float(example.value) if example.value is not None else None
|
|
51
|
+
if expected is None:
|
|
52
|
+
return None
|
|
53
|
+
if predicted is None:
|
|
54
|
+
return 0.0
|
|
55
|
+
return 1.0 if abs(predicted - expected) <= self._tolerance else 0.0
|
strop/metrics/types.py
ADDED
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
"""Core metric types."""
|
|
2
|
+
|
|
3
|
+
import json
|
|
4
|
+
from abc import ABC, abstractmethod
|
|
5
|
+
from collections.abc import Mapping, Sequence
|
|
6
|
+
from dataclasses import dataclass, field
|
|
7
|
+
from enum import StrEnum
|
|
8
|
+
from typing import Any
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
class Direction(StrEnum):
|
|
12
|
+
MAXIMIZE = "maximize"
|
|
13
|
+
MINIMIZE = "minimize"
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
class ErrorCode(StrEnum):
|
|
17
|
+
"""Deterministic failure categories attached to individual examples."""
|
|
18
|
+
|
|
19
|
+
PROVIDER_ERROR = "provider_error"
|
|
20
|
+
EMPTY_OUTPUT = "empty_output"
|
|
21
|
+
TRUNCATED = "truncated"
|
|
22
|
+
INVALID_FORMAT = "invalid_format"
|
|
23
|
+
SCHEMA_VIOLATION = "schema_violation"
|
|
24
|
+
WRONG_LABEL = "wrong_label"
|
|
25
|
+
WRONG_VALUE = "wrong_value"
|
|
26
|
+
MISSING_FIELD = "missing_field"
|
|
27
|
+
RULE_VIOLATION = "rule_violation"
|
|
28
|
+
LOW_SCORE = "low_score"
|
|
29
|
+
METRIC_ERROR = "metric_error"
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
@dataclass(frozen=True, slots=True)
|
|
33
|
+
class Example:
|
|
34
|
+
"""One model output paired with its row, ready to be scored."""
|
|
35
|
+
|
|
36
|
+
row_id: str
|
|
37
|
+
inputs: Mapping[str, str]
|
|
38
|
+
expected: str | None
|
|
39
|
+
raw_output: str
|
|
40
|
+
value: str | None = None
|
|
41
|
+
parsed: Any = None
|
|
42
|
+
error: str | None = None
|
|
43
|
+
|
|
44
|
+
@property
|
|
45
|
+
def valid(self) -> bool:
|
|
46
|
+
"""True if the output was produced and parsed under the output contract."""
|
|
47
|
+
return self.value is not None
|
|
48
|
+
|
|
49
|
+
@property
|
|
50
|
+
def correct(self) -> bool:
|
|
51
|
+
return self.value is not None and self.value == self.expected
|
|
52
|
+
|
|
53
|
+
@property
|
|
54
|
+
def expected_json(self) -> Any:
|
|
55
|
+
"""The expected output parsed as JSON, or None when absent or not JSON."""
|
|
56
|
+
if self.expected is None:
|
|
57
|
+
return None
|
|
58
|
+
try:
|
|
59
|
+
return json.loads(self.expected)
|
|
60
|
+
except ValueError:
|
|
61
|
+
return None
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
@dataclass(frozen=True)
|
|
65
|
+
class MetricValue:
|
|
66
|
+
"""A computed metric. ``value`` is None when it cannot be computed (no scorable examples)."""
|
|
67
|
+
|
|
68
|
+
name: str
|
|
69
|
+
value: float | None
|
|
70
|
+
samples: int
|
|
71
|
+
subjective: bool = False
|
|
72
|
+
details: dict[str, Any] = field(default_factory=dict)
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
class Metric(ABC):
|
|
76
|
+
"""A named score over a set of examples."""
|
|
77
|
+
|
|
78
|
+
name: str
|
|
79
|
+
direction: Direction = Direction.MAXIMIZE
|
|
80
|
+
subjective: bool = False
|
|
81
|
+
needs_expected: bool = True
|
|
82
|
+
|
|
83
|
+
@abstractmethod
|
|
84
|
+
def compute(self, examples: Sequence[Example]) -> MetricValue:
|
|
85
|
+
"""Aggregate score over ``examples``."""
|
|
86
|
+
|
|
87
|
+
async def prepare(self, examples: Sequence[Example]) -> None:
|
|
88
|
+
"""Do any batched or asynchronous work (model calls, subprocesses) before scoring."""
|
|
89
|
+
return None
|
|
90
|
+
|
|
91
|
+
def per_example(self, example: Example) -> float | None:
|
|
92
|
+
"""A per-example score used for paired comparison; None if not applicable.
|
|
93
|
+
|
|
94
|
+
The default is correctness (1 or 0), or None when there is no expected value.
|
|
95
|
+
Metrics that score on another scale override it.
|
|
96
|
+
"""
|
|
97
|
+
if example.expected is None:
|
|
98
|
+
return None
|
|
99
|
+
return 1.0 if example.correct else 0.0
|
|
100
|
+
|
|
101
|
+
def feedback(self, example: Example) -> str | None:
|
|
102
|
+
"""A short explanation of this example's score, shown to the optimizer."""
|
|
103
|
+
return None
|
|
104
|
+
|
|
105
|
+
def _result(self, value: float | None, samples: int, **details: Any) -> MetricValue:
|
|
106
|
+
return MetricValue(self.name, value, samples, self.subjective, dict(details))
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
"""Prompt optimization: proposing revisions and deciding whether to keep them."""
|
|
2
|
+
|
|
3
|
+
from strop.optimization.acceptance import AcceptanceRules, Decision, compare
|
|
4
|
+
from strop.optimization.generator import CandidateGenerator, OptimizerOutput, apply_output
|
|
5
|
+
from strop.optimization.types import (
|
|
6
|
+
Candidate,
|
|
7
|
+
HistoryEntry,
|
|
8
|
+
MetricSummary,
|
|
9
|
+
OptimizerView,
|
|
10
|
+
RejectedProposal,
|
|
11
|
+
)
|
|
12
|
+
|
|
13
|
+
__all__ = [
|
|
14
|
+
"AcceptanceRules",
|
|
15
|
+
"Candidate",
|
|
16
|
+
"CandidateGenerator",
|
|
17
|
+
"Decision",
|
|
18
|
+
"HistoryEntry",
|
|
19
|
+
"MetricSummary",
|
|
20
|
+
"OptimizerOutput",
|
|
21
|
+
"OptimizerView",
|
|
22
|
+
"RejectedProposal",
|
|
23
|
+
"apply_output",
|
|
24
|
+
"compare",
|
|
25
|
+
]
|