downshift 0.1.0.dev0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- downshift/__init__.py +3 -0
- downshift/audit.py +188 -0
- downshift/cli.py +755 -0
- downshift/config.py +348 -0
- downshift/cost.py +212 -0
- downshift/decide.py +220 -0
- downshift/evalgen.py +220 -0
- downshift/evals.py +395 -0
- downshift/llm.py +171 -0
- downshift/py.typed +0 -0
- downshift/report.py +380 -0
- downshift/resolve.py +439 -0
- downshift/runner.py +312 -0
- downshift/scanner.py +310 -0
- downshift/schema.py +356 -0
- downshift/scorer.py +219 -0
- downshift-0.1.0.dev0.dist-info/METADATA +24 -0
- downshift-0.1.0.dev0.dist-info/RECORD +21 -0
- downshift-0.1.0.dev0.dist-info/WHEEL +4 -0
- downshift-0.1.0.dev0.dist-info/entry_points.txt +2 -0
- downshift-0.1.0.dev0.dist-info/licenses/LICENSE +21 -0
downshift/schema.py
ADDED
|
@@ -0,0 +1,356 @@
|
|
|
1
|
+
"""Data format for LLM call sites (callsites.json).
|
|
2
|
+
|
|
3
|
+
The ast scanner writes it, the Bob auditor enriches it, and every later
|
|
4
|
+
command reads it. Everything goes through this module so both producers
|
|
5
|
+
are validated the same way.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import json
|
|
11
|
+
from collections.abc import Callable
|
|
12
|
+
from dataclasses import asdict, dataclass, field, fields
|
|
13
|
+
from pathlib import Path
|
|
14
|
+
from typing import Any
|
|
15
|
+
|
|
16
|
+
from downshift import __version__
|
|
17
|
+
|
|
18
|
+
SCHEMA_VERSION = 1
|
|
19
|
+
|
|
20
|
+
# How the scanner learned the model name.
|
|
21
|
+
MODEL_SOURCES = frozenset(
|
|
22
|
+
{
|
|
23
|
+
"literal", # model="gpt-4o" at the call
|
|
24
|
+
"constant", # a named constant, possibly imported from another module
|
|
25
|
+
"env_default", # os.getenv("X", "default"): the default is used
|
|
26
|
+
"env", # os.getenv("X") with no default: unknown until runtime
|
|
27
|
+
"dict_lookup", # MODELS["key"] on a dict literal
|
|
28
|
+
"parameter_default", # def f(model="gpt-4o")
|
|
29
|
+
"kwargs", # passed through **params and not resolvable
|
|
30
|
+
"missing", # no model argument at all
|
|
31
|
+
"dynamic", # anything else computed at runtime
|
|
32
|
+
"manual", # set by a human or by Bob
|
|
33
|
+
}
|
|
34
|
+
)
|
|
35
|
+
OUTPUT_FORMATS = frozenset({"text", "json"})
|
|
36
|
+
DIFFICULTIES = frozenset({"easy", "medium", "hard"})
|
|
37
|
+
GRADINGS = frozenset({"exact", "json_fields", "judge"})
|
|
38
|
+
PRODUCERS = frozenset({"ast", "bob", "manual"})
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
class SchemaError(ValueError):
|
|
42
|
+
"""Raised when callsites data does not match the schema."""
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
@dataclass
|
|
46
|
+
class ModelRef:
|
|
47
|
+
value: str | None
|
|
48
|
+
source: str
|
|
49
|
+
expression: str
|
|
50
|
+
env_var: str | None = None
|
|
51
|
+
defined_in: str | None = None
|
|
52
|
+
|
|
53
|
+
@property
|
|
54
|
+
def resolved(self) -> bool:
|
|
55
|
+
return self.value is not None
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
@dataclass
|
|
59
|
+
class PromptMessage:
|
|
60
|
+
role: str
|
|
61
|
+
content: str
|
|
62
|
+
resolved: bool = True
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
@dataclass
|
|
66
|
+
class CallSite:
|
|
67
|
+
# Where it is
|
|
68
|
+
id: str
|
|
69
|
+
file: str
|
|
70
|
+
line: int
|
|
71
|
+
function: str
|
|
72
|
+
api: str
|
|
73
|
+
model: ModelRef
|
|
74
|
+
end_line: int | None = None
|
|
75
|
+
is_async: bool = False
|
|
76
|
+
# What it sends
|
|
77
|
+
messages: list[PromptMessage] | None = None
|
|
78
|
+
output_format: str = "text"
|
|
79
|
+
temperature: float | None = None
|
|
80
|
+
max_tokens: int | None = None
|
|
81
|
+
# How it is reached
|
|
82
|
+
via: str | None = None
|
|
83
|
+
callers: list[str] = field(default_factory=list)
|
|
84
|
+
notes: list[str] = field(default_factory=list)
|
|
85
|
+
# Enrichment (Bob or a human); None when produced by the ast scanner
|
|
86
|
+
purpose: str | None = None
|
|
87
|
+
output_contract: str | None = None
|
|
88
|
+
difficulty: str | None = None
|
|
89
|
+
grading: str | None = None
|
|
90
|
+
found_by: str = "ast"
|
|
91
|
+
|
|
92
|
+
@property
|
|
93
|
+
def prompt_resolved(self) -> bool:
|
|
94
|
+
return self.messages is not None and all(m.resolved for m in self.messages)
|
|
95
|
+
|
|
96
|
+
def to_dict(self) -> dict[str, Any]:
|
|
97
|
+
return asdict(self)
|
|
98
|
+
|
|
99
|
+
@classmethod
|
|
100
|
+
def from_dict(cls, data: Any, path: str = "call_site") -> CallSite:
|
|
101
|
+
if not isinstance(data, dict):
|
|
102
|
+
raise SchemaError(f"{path}: expected an object, got {type(data).__name__}")
|
|
103
|
+
_reject_unknown(data, {f.name for f in fields(cls)}, path)
|
|
104
|
+
|
|
105
|
+
model_data = _field(data, "model", path, _is_dict, "an object")
|
|
106
|
+
return cls(
|
|
107
|
+
id=_field(data, "id", path, _is_nonempty_str, "a non-empty string"),
|
|
108
|
+
file=_field(data, "file", path, _is_nonempty_str, "a non-empty string"),
|
|
109
|
+
line=_field(data, "line", path, _is_positive_int, "a line number >= 1"),
|
|
110
|
+
function=_field(data, "function", path, _is_nonempty_str, "a non-empty string"),
|
|
111
|
+
api=_field(data, "api", path, _is_nonempty_str, "a non-empty string"),
|
|
112
|
+
model=_model_from_dict(model_data, f"{path}.model"),
|
|
113
|
+
end_line=_field(data, "end_line", path, _opt(_is_positive_int), "a line number", None),
|
|
114
|
+
is_async=_field(data, "is_async", path, _is_bool, "true or false", False),
|
|
115
|
+
messages=_messages_from_dict(data.get("messages"), f"{path}.messages"),
|
|
116
|
+
output_format=_field(
|
|
117
|
+
data,
|
|
118
|
+
"output_format",
|
|
119
|
+
path,
|
|
120
|
+
_one_of(OUTPUT_FORMATS),
|
|
121
|
+
_choices(OUTPUT_FORMATS),
|
|
122
|
+
"text",
|
|
123
|
+
),
|
|
124
|
+
temperature=_field(data, "temperature", path, _opt(_is_number), "a number", None),
|
|
125
|
+
max_tokens=_field(
|
|
126
|
+
data, "max_tokens", path, _opt(_is_positive_int), "an integer >= 1", None
|
|
127
|
+
),
|
|
128
|
+
via=_field(data, "via", path, _opt(_is_nonempty_str), "a call site id", None),
|
|
129
|
+
callers=_field(data, "callers", path, _is_str_list, "a list of strings", []),
|
|
130
|
+
notes=_field(data, "notes", path, _is_str_list, "a list of strings", []),
|
|
131
|
+
purpose=_field(data, "purpose", path, _opt(_is_str), "a string", None),
|
|
132
|
+
output_contract=_field(data, "output_contract", path, _opt(_is_str), "a string", None),
|
|
133
|
+
difficulty=_field(
|
|
134
|
+
data,
|
|
135
|
+
"difficulty",
|
|
136
|
+
path,
|
|
137
|
+
_opt(_one_of(DIFFICULTIES)),
|
|
138
|
+
_choices(DIFFICULTIES),
|
|
139
|
+
None,
|
|
140
|
+
),
|
|
141
|
+
grading=_field(
|
|
142
|
+
data,
|
|
143
|
+
"grading",
|
|
144
|
+
path,
|
|
145
|
+
_opt(_one_of(GRADINGS)),
|
|
146
|
+
_choices(GRADINGS),
|
|
147
|
+
None,
|
|
148
|
+
),
|
|
149
|
+
found_by=_field(
|
|
150
|
+
data,
|
|
151
|
+
"found_by",
|
|
152
|
+
path,
|
|
153
|
+
_one_of(PRODUCERS),
|
|
154
|
+
_choices(PRODUCERS),
|
|
155
|
+
"ast",
|
|
156
|
+
),
|
|
157
|
+
)
|
|
158
|
+
|
|
159
|
+
|
|
160
|
+
@dataclass
|
|
161
|
+
class ScanResult:
|
|
162
|
+
root: str
|
|
163
|
+
files_scanned: int
|
|
164
|
+
call_sites: list[CallSite]
|
|
165
|
+
warnings: list[str] = field(default_factory=list)
|
|
166
|
+
generated_by: str = "ast"
|
|
167
|
+
tool_version: str = __version__
|
|
168
|
+
|
|
169
|
+
def summary(self) -> dict[str, int]:
|
|
170
|
+
return {
|
|
171
|
+
"files_scanned": self.files_scanned,
|
|
172
|
+
"call_sites": len(self.call_sites),
|
|
173
|
+
"models_resolved": sum(1 for s in self.call_sites if s.model.resolved),
|
|
174
|
+
"prompts_resolved": sum(1 for s in self.call_sites if s.prompt_resolved),
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
def to_dict(self) -> dict[str, Any]:
|
|
178
|
+
return {
|
|
179
|
+
"schema_version": SCHEMA_VERSION,
|
|
180
|
+
"tool": "downshift",
|
|
181
|
+
"tool_version": self.tool_version,
|
|
182
|
+
"generated_by": self.generated_by,
|
|
183
|
+
"root": self.root,
|
|
184
|
+
"summary": self.summary(),
|
|
185
|
+
"warnings": list(self.warnings),
|
|
186
|
+
"call_sites": [site.to_dict() for site in self.call_sites],
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
def to_json(self) -> str:
|
|
190
|
+
return json.dumps(self.to_dict(), indent=2, ensure_ascii=False) + "\n"
|
|
191
|
+
|
|
192
|
+
def write(self, path: Path) -> None:
|
|
193
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
194
|
+
path.write_text(self.to_json(), encoding="utf-8")
|
|
195
|
+
|
|
196
|
+
@classmethod
|
|
197
|
+
def from_dict(cls, data: Any) -> ScanResult:
|
|
198
|
+
if not isinstance(data, dict):
|
|
199
|
+
raise SchemaError("callsites: expected a JSON object at the top level")
|
|
200
|
+
version = data.get("schema_version")
|
|
201
|
+
if version != SCHEMA_VERSION:
|
|
202
|
+
raise SchemaError(
|
|
203
|
+
f"schema_version: unsupported value {version!r} (expected {SCHEMA_VERSION})"
|
|
204
|
+
)
|
|
205
|
+
raw_sites = data.get("call_sites")
|
|
206
|
+
if not isinstance(raw_sites, list):
|
|
207
|
+
raise SchemaError("call_sites: expected a list")
|
|
208
|
+
sites = [CallSite.from_dict(item, f"call_sites[{i}]") for i, item in enumerate(raw_sites)]
|
|
209
|
+
|
|
210
|
+
seen: set[str] = set()
|
|
211
|
+
for site in sites:
|
|
212
|
+
if site.id in seen:
|
|
213
|
+
raise SchemaError(f"call_sites: duplicate id {site.id!r}")
|
|
214
|
+
seen.add(site.id)
|
|
215
|
+
|
|
216
|
+
return cls(
|
|
217
|
+
root=_field(data, "root", "callsites", _is_str, "a string", "."),
|
|
218
|
+
files_scanned=_summary_files(data),
|
|
219
|
+
call_sites=sites,
|
|
220
|
+
warnings=_field(data, "warnings", "callsites", _is_str_list, "a list of strings", []),
|
|
221
|
+
generated_by=_field(
|
|
222
|
+
data,
|
|
223
|
+
"generated_by",
|
|
224
|
+
"callsites",
|
|
225
|
+
_one_of(PRODUCERS),
|
|
226
|
+
_choices(PRODUCERS),
|
|
227
|
+
"ast",
|
|
228
|
+
),
|
|
229
|
+
tool_version=_field(
|
|
230
|
+
data, "tool_version", "callsites", _is_str, "a string", __version__
|
|
231
|
+
),
|
|
232
|
+
)
|
|
233
|
+
|
|
234
|
+
@classmethod
|
|
235
|
+
def load(cls, path: Path) -> ScanResult:
|
|
236
|
+
try:
|
|
237
|
+
text = path.read_text(encoding="utf-8")
|
|
238
|
+
except FileNotFoundError:
|
|
239
|
+
raise SchemaError(f"callsites file not found: {path}") from None
|
|
240
|
+
try:
|
|
241
|
+
data = json.loads(text)
|
|
242
|
+
except json.JSONDecodeError as exc:
|
|
243
|
+
raise SchemaError(f"{path}: invalid JSON: {exc}") from exc
|
|
244
|
+
return cls.from_dict(data)
|
|
245
|
+
|
|
246
|
+
|
|
247
|
+
# --- helpers ------------------------------------------------------------------
|
|
248
|
+
|
|
249
|
+
_MISSING: Any = object()
|
|
250
|
+
|
|
251
|
+
|
|
252
|
+
def _field(
|
|
253
|
+
data: dict[str, Any],
|
|
254
|
+
key: str,
|
|
255
|
+
path: str,
|
|
256
|
+
check: Callable[[Any], bool],
|
|
257
|
+
expected: str,
|
|
258
|
+
default: Any = _MISSING,
|
|
259
|
+
) -> Any:
|
|
260
|
+
if key not in data:
|
|
261
|
+
if default is _MISSING:
|
|
262
|
+
raise SchemaError(f"{path}.{key}: missing required field")
|
|
263
|
+
return default
|
|
264
|
+
value = data[key]
|
|
265
|
+
if not check(value):
|
|
266
|
+
raise SchemaError(f"{path}.{key}: expected {expected}, got {value!r}")
|
|
267
|
+
return value
|
|
268
|
+
|
|
269
|
+
|
|
270
|
+
def _reject_unknown(data: dict[str, Any], allowed: set[str], path: str) -> None:
|
|
271
|
+
unknown = sorted(set(data) - allowed)
|
|
272
|
+
if unknown:
|
|
273
|
+
raise SchemaError(f"{path}: unknown field(s) {', '.join(unknown)}")
|
|
274
|
+
|
|
275
|
+
|
|
276
|
+
def _model_from_dict(data: dict[str, Any], path: str) -> ModelRef:
|
|
277
|
+
_reject_unknown(data, {f.name for f in fields(ModelRef)}, path)
|
|
278
|
+
return ModelRef(
|
|
279
|
+
value=_field(data, "value", path, _opt(_is_nonempty_str), "a model name or null"),
|
|
280
|
+
source=_field(data, "source", path, _one_of(MODEL_SOURCES), _choices(MODEL_SOURCES)),
|
|
281
|
+
expression=_field(data, "expression", path, _is_str, "a string"),
|
|
282
|
+
env_var=_field(data, "env_var", path, _opt(_is_nonempty_str), "a string", None),
|
|
283
|
+
defined_in=_field(data, "defined_in", path, _opt(_is_nonempty_str), "a string", None),
|
|
284
|
+
)
|
|
285
|
+
|
|
286
|
+
|
|
287
|
+
def _messages_from_dict(data: Any, path: str) -> list[PromptMessage] | None:
|
|
288
|
+
if data is None:
|
|
289
|
+
return None
|
|
290
|
+
if not isinstance(data, list):
|
|
291
|
+
raise SchemaError(f"{path}: expected a list or null")
|
|
292
|
+
messages = []
|
|
293
|
+
for i, item in enumerate(data):
|
|
294
|
+
item_path = f"{path}[{i}]"
|
|
295
|
+
if not isinstance(item, dict):
|
|
296
|
+
raise SchemaError(f"{item_path}: expected an object")
|
|
297
|
+
_reject_unknown(item, {f.name for f in fields(PromptMessage)}, item_path)
|
|
298
|
+
messages.append(
|
|
299
|
+
PromptMessage(
|
|
300
|
+
role=_field(item, "role", item_path, _is_nonempty_str, "a non-empty string"),
|
|
301
|
+
content=_field(item, "content", item_path, _is_str, "a string"),
|
|
302
|
+
resolved=_field(item, "resolved", item_path, _is_bool, "true or false", True),
|
|
303
|
+
)
|
|
304
|
+
)
|
|
305
|
+
return messages
|
|
306
|
+
|
|
307
|
+
|
|
308
|
+
def _summary_files(data: dict[str, Any]) -> int:
|
|
309
|
+
summary = data.get("summary")
|
|
310
|
+
if isinstance(summary, dict) and _is_nonneg_int(summary.get("files_scanned")):
|
|
311
|
+
return int(summary["files_scanned"])
|
|
312
|
+
return 0
|
|
313
|
+
|
|
314
|
+
|
|
315
|
+
def _choices(options: frozenset[str]) -> str:
|
|
316
|
+
return "one of " + ", ".join(sorted(options))
|
|
317
|
+
|
|
318
|
+
|
|
319
|
+
def _is_str(value: Any) -> bool:
|
|
320
|
+
return isinstance(value, str)
|
|
321
|
+
|
|
322
|
+
|
|
323
|
+
def _is_nonempty_str(value: Any) -> bool:
|
|
324
|
+
return isinstance(value, str) and value.strip() != ""
|
|
325
|
+
|
|
326
|
+
|
|
327
|
+
def _is_bool(value: Any) -> bool:
|
|
328
|
+
return isinstance(value, bool)
|
|
329
|
+
|
|
330
|
+
|
|
331
|
+
def _is_dict(value: Any) -> bool:
|
|
332
|
+
return isinstance(value, dict)
|
|
333
|
+
|
|
334
|
+
|
|
335
|
+
def _is_number(value: Any) -> bool:
|
|
336
|
+
return isinstance(value, (int, float)) and not isinstance(value, bool)
|
|
337
|
+
|
|
338
|
+
|
|
339
|
+
def _is_positive_int(value: Any) -> bool:
|
|
340
|
+
return isinstance(value, int) and not isinstance(value, bool) and value >= 1
|
|
341
|
+
|
|
342
|
+
|
|
343
|
+
def _is_nonneg_int(value: Any) -> bool:
|
|
344
|
+
return isinstance(value, int) and not isinstance(value, bool) and value >= 0
|
|
345
|
+
|
|
346
|
+
|
|
347
|
+
def _is_str_list(value: Any) -> bool:
|
|
348
|
+
return isinstance(value, list) and all(isinstance(v, str) for v in value)
|
|
349
|
+
|
|
350
|
+
|
|
351
|
+
def _opt(check: Callable[[Any], bool]) -> Callable[[Any], bool]:
|
|
352
|
+
return lambda value: value is None or check(value)
|
|
353
|
+
|
|
354
|
+
|
|
355
|
+
def _one_of(options: frozenset[str]) -> Callable[[Any], bool]:
|
|
356
|
+
return lambda value: isinstance(value, str) and value in options
|
downshift/scorer.py
ADDED
|
@@ -0,0 +1,219 @@
|
|
|
1
|
+
"""Score model outputs against eval expectations.
|
|
2
|
+
|
|
3
|
+
Three gradings, matching the call site's `grading` field:
|
|
4
|
+
exact strict string match after normalizing case, whitespace, fences
|
|
5
|
+
and outer punctuation
|
|
6
|
+
json_fields output parsed as a JSON object, graded fields compared one by one
|
|
7
|
+
judge a judge model scores the output 1 to 5 against the case rubric
|
|
8
|
+
|
|
9
|
+
Every scorer returns a Score with a value in [0, 1] and a pass flag, so the
|
|
10
|
+
decide step can treat all gradings the same way.
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
import json
|
|
16
|
+
import re
|
|
17
|
+
from collections.abc import Mapping, Sequence
|
|
18
|
+
from dataclasses import asdict, dataclass
|
|
19
|
+
from typing import Any
|
|
20
|
+
|
|
21
|
+
from downshift.llm import ChatMessage, LLMClient
|
|
22
|
+
|
|
23
|
+
JUDGE_PASS_AT = 4
|
|
24
|
+
|
|
25
|
+
_OUTER_PUNCT = " \t\r\n.,;:!?\"'`*()[]"
|
|
26
|
+
_FENCE = re.compile(r"^```(?:[\w-]+\n)?\s*(.*?)\s*```$", re.DOTALL)
|
|
27
|
+
_SCORE_LABEL = re.compile(r"\bscore\b\s*(?:is|of|=|:)?\s*\**\s*([1-5])(?!\d|\.\d)", re.IGNORECASE)
|
|
28
|
+
_OUT_OF = re.compile(r"(?<![\d.])([1-5])\s*(?:/|out of)\s*5(?!\d)", re.IGNORECASE)
|
|
29
|
+
_LONE = re.compile(r"\s*([1-5])\s*\.?\s*")
|
|
30
|
+
|
|
31
|
+
JUDGE_SYSTEM = (
|
|
32
|
+
"You grade the output of an AI assistant inside a customer support app. "
|
|
33
|
+
"Read the task the assistant was given, its output, and the rubric. "
|
|
34
|
+
"Score the output from 1 (fails the rubric) to 5 (fully meets it). "
|
|
35
|
+
'Reply with JSON only: {"score": <1-5>, "reason": "<one sentence>"}.'
|
|
36
|
+
)
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
@dataclass(frozen=True)
|
|
40
|
+
class Score:
|
|
41
|
+
"""Result of scoring one output. value is in [0, 1]."""
|
|
42
|
+
|
|
43
|
+
value: float
|
|
44
|
+
passed: bool
|
|
45
|
+
detail: str = ""
|
|
46
|
+
judge_score: int | None = None
|
|
47
|
+
|
|
48
|
+
def to_dict(self) -> dict[str, Any]:
|
|
49
|
+
return asdict(self)
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
@dataclass(frozen=True)
|
|
53
|
+
class Judge:
|
|
54
|
+
"""The model that grades judge-graded call sites."""
|
|
55
|
+
|
|
56
|
+
client: LLMClient
|
|
57
|
+
model: str
|
|
58
|
+
pass_at: int = JUDGE_PASS_AT
|
|
59
|
+
max_tokens: int = 200
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def _clip(text: str, limit: int = 80) -> str:
|
|
63
|
+
return text if len(text) <= limit else text[: limit - 3] + "..."
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def strip_fences(text: str) -> str:
|
|
67
|
+
"""Remove a surrounding ``` or ```lang fence, if any."""
|
|
68
|
+
stripped = text.strip()
|
|
69
|
+
match = _FENCE.match(stripped)
|
|
70
|
+
return match.group(1).strip() if match else stripped
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def normalize_text(text: str) -> str:
|
|
74
|
+
"""Lowercase, collapse whitespace, drop fences and outer punctuation."""
|
|
75
|
+
collapsed = " ".join(strip_fences(text).split()).lower()
|
|
76
|
+
return collapsed.strip(_OUTER_PUNCT)
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def score_exact(output: str, expected: Any) -> Score:
|
|
80
|
+
got = normalize_text(output)
|
|
81
|
+
want = normalize_text(str(expected))
|
|
82
|
+
if got == want:
|
|
83
|
+
return Score(1.0, True, "match")
|
|
84
|
+
return Score(0.0, False, f"expected {want!r}, got {_clip(got)!r}")
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def parse_json_object(text: str) -> dict[str, Any] | None:
|
|
88
|
+
"""Parse a JSON object from model output (fences and surrounding prose allowed)."""
|
|
89
|
+
body = strip_fences(text)
|
|
90
|
+
try:
|
|
91
|
+
data = json.loads(body)
|
|
92
|
+
except json.JSONDecodeError:
|
|
93
|
+
start, end = body.find("{"), body.rfind("}")
|
|
94
|
+
if start == -1 or end <= start:
|
|
95
|
+
return None
|
|
96
|
+
try:
|
|
97
|
+
data = json.loads(body[start : end + 1])
|
|
98
|
+
except json.JSONDecodeError:
|
|
99
|
+
return None
|
|
100
|
+
return data if isinstance(data, dict) else None
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
def _norm_value(value: Any) -> str:
|
|
104
|
+
if value is None:
|
|
105
|
+
return ""
|
|
106
|
+
if isinstance(value, bool):
|
|
107
|
+
return "true" if value else "false"
|
|
108
|
+
if isinstance(value, float) and value.is_integer():
|
|
109
|
+
value = int(value)
|
|
110
|
+
if isinstance(value, dict | list):
|
|
111
|
+
return json.dumps(value, sort_keys=True).lower()
|
|
112
|
+
text = " ".join(str(value).split()).lower()
|
|
113
|
+
return "" if text in {"null", "none"} else text
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
def score_json_fields(
|
|
117
|
+
output: str, expected: Mapping[str, Any], fields: Sequence[str] | None = None
|
|
118
|
+
) -> Score:
|
|
119
|
+
"""Compare graded fields. value = fraction matching; passed only if all match."""
|
|
120
|
+
names = list(fields) if fields else list(expected)
|
|
121
|
+
if not names:
|
|
122
|
+
raise ValueError("json_fields grading needs at least one field")
|
|
123
|
+
data = parse_json_object(output)
|
|
124
|
+
if data is None:
|
|
125
|
+
return Score(0.0, False, "output is not a JSON object")
|
|
126
|
+
wrong: list[str] = []
|
|
127
|
+
for name in names:
|
|
128
|
+
if name not in data:
|
|
129
|
+
wrong.append(f"{name}: missing")
|
|
130
|
+
elif _norm_value(data[name]) != _norm_value(expected.get(name)):
|
|
131
|
+
wrong.append(f"{name}: expected {expected.get(name)!r}, got {data[name]!r}")
|
|
132
|
+
value = (len(names) - len(wrong)) / len(names)
|
|
133
|
+
detail = "all fields match" if not wrong else "; ".join(wrong)
|
|
134
|
+
return Score(value, not wrong, detail)
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
def _as_score(raw: Any) -> int | None:
|
|
138
|
+
if isinstance(raw, bool):
|
|
139
|
+
return None
|
|
140
|
+
try:
|
|
141
|
+
number = float(raw)
|
|
142
|
+
except (TypeError, ValueError):
|
|
143
|
+
return None
|
|
144
|
+
if not number.is_integer() or not 1 <= number <= 5:
|
|
145
|
+
return None
|
|
146
|
+
return int(number)
|
|
147
|
+
|
|
148
|
+
|
|
149
|
+
def parse_judge_score(text: str) -> int | None:
|
|
150
|
+
"""Pull a 1 to 5 score out of a judge reply. None if there is no clear score."""
|
|
151
|
+
data = parse_json_object(text)
|
|
152
|
+
if data is not None and "score" in data:
|
|
153
|
+
return _as_score(data["score"])
|
|
154
|
+
for pattern in (_SCORE_LABEL, _OUT_OF):
|
|
155
|
+
match = pattern.search(text)
|
|
156
|
+
if match:
|
|
157
|
+
return int(match.group(1))
|
|
158
|
+
lone = _LONE.fullmatch(text)
|
|
159
|
+
return int(lone.group(1)) if lone else None
|
|
160
|
+
|
|
161
|
+
|
|
162
|
+
def build_judge_messages(
|
|
163
|
+
prompt_messages: Sequence[ChatMessage], output: str, rubric: str
|
|
164
|
+
) -> list[dict[str, str]]:
|
|
165
|
+
task = "\n\n".join(
|
|
166
|
+
f"[{m.get('role', 'user')}]\n{m.get('content', '')}" for m in prompt_messages
|
|
167
|
+
)
|
|
168
|
+
user = (
|
|
169
|
+
f"## Task given to the assistant\n{task}\n\n"
|
|
170
|
+
f"## Assistant output\n{output}\n\n"
|
|
171
|
+
f"## Rubric\n{rubric}\n\n"
|
|
172
|
+
"Return the JSON now."
|
|
173
|
+
)
|
|
174
|
+
return [{"role": "system", "content": JUDGE_SYSTEM}, {"role": "user", "content": user}]
|
|
175
|
+
|
|
176
|
+
|
|
177
|
+
def score_judge(
|
|
178
|
+
judge: Judge, prompt_messages: Sequence[ChatMessage], output: str, rubric: str
|
|
179
|
+
) -> Score:
|
|
180
|
+
"""Ask the judge model for a 1 to 5 score. LLMError propagates to the caller."""
|
|
181
|
+
if not output.strip():
|
|
182
|
+
return Score(0.0, False, "empty output")
|
|
183
|
+
completion = judge.client.complete(
|
|
184
|
+
judge.model,
|
|
185
|
+
build_judge_messages(prompt_messages, output, rubric),
|
|
186
|
+
temperature=0.0,
|
|
187
|
+
max_tokens=judge.max_tokens,
|
|
188
|
+
json_mode=True,
|
|
189
|
+
)
|
|
190
|
+
score = parse_judge_score(completion.text)
|
|
191
|
+
if score is None:
|
|
192
|
+
return Score(0.0, False, f"judge reply unparseable: {_clip(completion.text)!r}")
|
|
193
|
+
data = parse_json_object(completion.text)
|
|
194
|
+
reason = str(data.get("reason", "")).strip() if data else ""
|
|
195
|
+
detail = f"judge {score}/5" + (f": {_clip(reason, 160)}" if reason else "")
|
|
196
|
+
return Score((score - 1) / 4, score >= judge.pass_at, detail, judge_score=score)
|
|
197
|
+
|
|
198
|
+
|
|
199
|
+
def score_case(
|
|
200
|
+
grading: str,
|
|
201
|
+
expected: Any,
|
|
202
|
+
output: str,
|
|
203
|
+
*,
|
|
204
|
+
fields: Sequence[str] | None = None,
|
|
205
|
+
prompt_messages: Sequence[ChatMessage] = (),
|
|
206
|
+
judge: Judge | None = None,
|
|
207
|
+
) -> Score:
|
|
208
|
+
"""Score one output with the case's grading."""
|
|
209
|
+
if grading == "exact":
|
|
210
|
+
return score_exact(output, expected)
|
|
211
|
+
if grading == "json_fields":
|
|
212
|
+
if not isinstance(expected, Mapping):
|
|
213
|
+
raise ValueError("json_fields expected value must be an object")
|
|
214
|
+
return score_json_fields(output, expected, fields)
|
|
215
|
+
if grading == "judge":
|
|
216
|
+
if judge is None:
|
|
217
|
+
raise ValueError("judge grading needs a Judge")
|
|
218
|
+
return score_judge(judge, prompt_messages, output, str(expected))
|
|
219
|
+
raise ValueError(f"unknown grading {grading!r}")
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: downshift
|
|
3
|
+
Version: 0.1.0.dev0
|
|
4
|
+
Summary: Cut LLM costs per PR: find every LLM call, prove which can use cheaper models, and show the cost impact of every change.
|
|
5
|
+
Author: Anagha
|
|
6
|
+
License: MIT
|
|
7
|
+
License-File: LICENSE
|
|
8
|
+
Requires-Python: >=3.10
|
|
9
|
+
Requires-Dist: openai>=1.40
|
|
10
|
+
Requires-Dist: pyyaml>=6.0
|
|
11
|
+
Requires-Dist: rich>=13.0
|
|
12
|
+
Requires-Dist: tiktoken>=0.7
|
|
13
|
+
Requires-Dist: typer>=0.12
|
|
14
|
+
Provides-Extra: dev
|
|
15
|
+
Requires-Dist: build>=1.2; extra == 'dev'
|
|
16
|
+
Requires-Dist: mypy>=1.10; extra == 'dev'
|
|
17
|
+
Requires-Dist: pre-commit>=3.7; extra == 'dev'
|
|
18
|
+
Requires-Dist: pytest-cov>=5.0; extra == 'dev'
|
|
19
|
+
Requires-Dist: pytest>=8.0; extra == 'dev'
|
|
20
|
+
Requires-Dist: ruff>=0.6; extra == 'dev'
|
|
21
|
+
Requires-Dist: types-pyyaml; extra == 'dev'
|
|
22
|
+
Description-Content-Type: text/markdown
|
|
23
|
+
|
|
24
|
+
# downshift
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
downshift/__init__.py,sha256=yDxO5m74Yt4x2i2ppmLKlJNiayqn_H_UnICVGU1Vam4,67
|
|
2
|
+
downshift/audit.py,sha256=_0KpdjGx-N0JaXX4CoU3mvWkqDwj6zrUwWb9lPvS4To,6749
|
|
3
|
+
downshift/cli.py,sha256=uof5rNFfeBvmv4Fxa4CzhL_iUDrS-5uS5kGV0bw60UI,26863
|
|
4
|
+
downshift/config.py,sha256=SjAV4v2AGEpanDkPeVLc2k-RXW1ToP1CxLKprp376hU,12480
|
|
5
|
+
downshift/cost.py,sha256=xRMOKCALznb8S5cmHpdEr-5Rz-KA6RCDgr5noMiEiN4,6778
|
|
6
|
+
downshift/decide.py,sha256=J_g40peYe7cTvwGBSmk2p1nJn8JAW8ZebFTTxN7KgJ8,7356
|
|
7
|
+
downshift/evalgen.py,sha256=-Rw5kTYQkCaMdwd0nElYqb45AE9DB3wuI8YCqx5jdw4,7891
|
|
8
|
+
downshift/evals.py,sha256=Tk3OLu41EEvMUe8a6xpyTFo0pCh-GVc_iQ4jtXUNMZs,14063
|
|
9
|
+
downshift/llm.py,sha256=TNnCdMlt-7r_GJqiWnv2scGf_WbKRHT_SdZRzK2fvxk,5022
|
|
10
|
+
downshift/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
11
|
+
downshift/report.py,sha256=5IFDLEcuRePW6J_ySortMEJBbJ72Wf5vYFhdyyjKYz0,13395
|
|
12
|
+
downshift/resolve.py,sha256=AarZ2vfBFGnky9514MzV839-ecCDHS0wY6_ZrBzt578,17170
|
|
13
|
+
downshift/runner.py,sha256=UoN78kGOFRr0UXAbodVsCMY9ViSc8vS1r84doFvzjBM,10601
|
|
14
|
+
downshift/scanner.py,sha256=rXCMUYUAyQ9fspJYrKfkT9MrHDO5trxbc74rWh1ud_8,11011
|
|
15
|
+
downshift/schema.py,sha256=LCXCyBD8fRouVI3sLL8SnchpUOTXbxHrOrYaJMZ1lHQ,12270
|
|
16
|
+
downshift/scorer.py,sha256=_tN5Fwz5P6rtQUJxoZdkkqUSkBLr105xXyhrKnazYvo,7566
|
|
17
|
+
downshift-0.1.0.dev0.dist-info/METADATA,sha256=QYJp2ueYTnFqmZ7LY_W1EF7FDexQ_V_ZSQ65pfE2J60,777
|
|
18
|
+
downshift-0.1.0.dev0.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
|
|
19
|
+
downshift-0.1.0.dev0.dist-info/entry_points.txt,sha256=ycLci7nmvEnA_ieGWyiso5hqd3DzK4SkOBg2YrOtGPg,49
|
|
20
|
+
downshift-0.1.0.dev0.dist-info/licenses/LICENSE,sha256=tYT1Vf4Zel79dY5o7HQuPQ5mIxMXow60cgmh_J3mbIc,1077
|
|
21
|
+
downshift-0.1.0.dev0.dist-info/RECORD,,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Anagha Hambir Langhe
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|