traceground 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,26 @@
1
+ """traceground: checks an AI answer against its sources or its tool-call trace.
2
+
3
+ from traceground import check_answer
4
+ result = check_answer(answer, sources=["...", "..."], question="...")
5
+ print(result.verdict, result.summary)
6
+ for f in result.findings:
7
+ print(f.text, f.message)
8
+
9
+ # or with a tool-call trace: a list of {"name", "args", "result"}, or OpenAI-style chat messages
10
+ result = check_answer(answer, question="...", trace=[{"name": "loan_cost", "args": {...}, "result": {...}}])
11
+ result = check_messages(messages)
12
+
13
+ In development (0.1.0.dev): numbers, quotes, value-to-field and argument binding, failed calls, tools
14
+ never called, thinking aloud and self-contradiction are implemented. See https://github.com/animeshQKD/traceground.
15
+ """
16
+ from importlib.metadata import PackageNotFoundError, version
17
+
18
+ from traceground.api import check_answer, check_messages, normalize_sources
19
+ from traceground.result import Finding, Result
20
+
21
+ try:
22
+ __version__ = version("traceground")
23
+ except PackageNotFoundError: # running from a source tree that was not installed
24
+ __version__ = "0.0.0+unknown"
25
+
26
+ __all__ = ["__version__", "check_answer", "check_messages", "normalize_sources", "Finding", "Result"]
@@ -0,0 +1,6 @@
1
+ """python -m traceground"""
2
+ import sys
3
+
4
+ from traceground.cli import main
5
+
6
+ sys.exit(main())
traceground/api.py ADDED
@@ -0,0 +1,187 @@
1
+ """check_answer() and check_messages(): the calls most users need."""
2
+ from traceground import binding, numbers, quotes, selftalk, textnorm, toolchecks, traces
3
+ from traceground.result import Result
4
+
5
+
6
+ def normalize_sources(sources) -> list:
7
+ """Accept the usual shapes and return [(id, text), ...]:
8
+ None; one string; a list of strings (ids S1, S2, ...); a dict {id: text}; a list of
9
+ (id, text) pairs; a list of dicts with "text" (and "id")."""
10
+ if sources is None:
11
+ return []
12
+ if isinstance(sources, str):
13
+ return [("S1", sources)]
14
+ if isinstance(sources, dict):
15
+ return _checked_ids([(str(k), str(v)) for k, v in sources.items()])
16
+ out = []
17
+ for i, s in enumerate(sources, 1):
18
+ if isinstance(s, str):
19
+ out.append((f"S{i}", s))
20
+ elif isinstance(s, dict):
21
+ out.append((str(s.get("id", f"S{i}")), str(s["text"])))
22
+ elif isinstance(s, (tuple, list)) and len(s) == 2:
23
+ out.append((str(s[0]), str(s[1])))
24
+ else:
25
+ raise TypeError(f"source {i}: expected a string, a (id, text) pair or a dict with 'text', got {type(s).__name__}")
26
+ return _checked_ids(out)
27
+
28
+
29
+ def _checked_ids(out: list) -> list:
30
+ ids = [sid for sid, _t in out]
31
+ if len(set(ids)) != len(ids):
32
+ raise ValueError(f"source ids must be unique, got {ids}")
33
+ if "question" in ids:
34
+ raise ValueError('"question" is reserved for the question; give the source another id')
35
+ return out
36
+
37
+
38
+ def check_answer(answer, sources=None, question: str = "", trace=None, *, percent_as_fraction: bool = True,
39
+ check_small_integers: bool = False, check_quotes: bool = True, min_quote_words: int = 3,
40
+ check_bindings: bool = True, field_synonyms=None, check_wording: bool = True) -> Result:
41
+ """Check `answer` against its evidence: `sources` (texts), the `question`, and a tool-call `trace`.
42
+
43
+ Checks: every number must be supported by the evidence; every quotation of `min_quote_words` words
44
+ or more must occur in the source its sentence cites (or in any source). With a trace, also: a value
45
+ the answer attaches to a field name must be the value the tool gave for it (value-to-field); the
46
+ values in the question must be the ones the tools were called with (argument binding); a failed
47
+ call must not be reported as a result; the answer must not claim tools that were not called.
48
+ The wording (check_wording): no thinking aloud left in, and no "the sources do not say X" followed
49
+ by X from the sources.
50
+
51
+ `trace` is a list of {"name", "args", "result"} or OpenAI-style chat messages (see traces.py). When
52
+ the messages hold the question and the final answer, `answer` and `question` may be left empty.
53
+
54
+ Returns a Result: verdict "ok", "problems" (at least one finding of severity "high") or "unsure"
55
+ (there are numbers or quotes to check but no evidence at all, and nothing else was found). Nothing
56
+ in the answer is changed.
57
+ """
58
+ calls = []
59
+ if trace is not None:
60
+ t_question, calls, t_answer = traces.load_trace(trace)
61
+ question = question or t_question
62
+ answer = answer if answer else t_answer
63
+ traces.name_calls(calls)
64
+ if trace is not None and not answer:
65
+ raise ValueError("no answer to check: pass the answer, or a chat whose last assistant message is the answer")
66
+ if not isinstance(answer, str):
67
+ raise TypeError(f"answer must be a string, got {type(answer).__name__}")
68
+ srcs = normalize_sources(sources)
69
+ question = question or ""
70
+ ids = {sid for sid, _t in srcs}
71
+ clash = [c.id for c in calls if c.id in ids or c.id == "question"]
72
+ if clash:
73
+ raise ValueError(f"a tool call and a source have the same id: {clash}")
74
+ evidence = srcs + [(c.id, c.render() + _counts_text(c)) for c in calls]
75
+ spans = textnorm.sentence_spans(answer)
76
+
77
+ def sentence_of(pos):
78
+ return textnorm.sentence_at(answer, pos, spans)
79
+
80
+ has_q = bool(question.strip())
81
+ has_evidence = bool(srcs or calls or has_q)
82
+ kinds = (["the sources"] if srcs else []) + (["the tool results"] if calls else []) + ["the question"]
83
+ name = ", ".join(kinds[:-1]) + " or " + kinds[-1] if len(kinds) > 1 else kinds[0]
84
+ findings, stats = numbers.check_numbers(
85
+ answer, evidence, question, percent_as_fraction=percent_as_fraction,
86
+ check_small_integers=check_small_integers, sentence_of=sentence_of, evidence_name=name)
87
+ if check_quotes:
88
+ q_findings, q_stats = quotes.check_quotes(answer, evidence, question, min_words=min_quote_words,
89
+ sentence_of=sentence_of)
90
+ inside = [(f.start, f.end) for f in q_findings]
91
+ findings = [f for f in findings if not any(a <= f.start < b for a, b in inside)] + q_findings
92
+ stats.update(q_stats)
93
+ else:
94
+ stats.update({"quotes_checked": 0, "quotes_not_found": 0, "quotes_wrong_source": 0,
95
+ "quotes_changed_slightly": 0})
96
+ if not has_evidence:
97
+ findings = [] # nothing to check numbers and quotes against: no verdict on them
98
+ stats.update({"bindings_checked": 0, "bindings_wrong": 0, "question_values": 0, "arguments_wrong": 0,
99
+ "tool_calls_failed_unreported": 0, "tool_claims_unbacked": 0})
100
+ if trace is not None:
101
+ if calls and check_bindings:
102
+ v_findings, v_stats = binding.check_value_to_field(answer, calls, field_synonyms=field_synonyms,
103
+ sentence_of=sentence_of)
104
+ bound = [(f.start, f.end) for f in v_findings]
105
+ findings = [f for f in findings if not any(a < f.end and f.start < b for a, b in bound)] + v_findings
106
+ a_findings, a_stats = binding.check_arguments(answer, question, calls, sentence_spans=spans,
107
+ sentence_of=sentence_of)
108
+ findings += a_findings
109
+ stats.update(v_stats)
110
+ stats.update(a_stats)
111
+ f_findings, f_stats = toolchecks.check_failed_calls(answer, calls, sentence_spans=spans,
112
+ sentence_of=sentence_of)
113
+ n_findings, n_stats = toolchecks.check_tools_not_called(answer, calls, sentence_of=sentence_of)
114
+ findings += f_findings + n_findings
115
+ stats.update(f_stats)
116
+ stats.update(n_stats)
117
+ stats.update({"self_talk": 0, "self_contradictions": 0})
118
+ if check_wording:
119
+ s_findings, s_stats = selftalk.check_self_talk(answer, sentence_of=sentence_of)
120
+ c_findings, c_stats = selftalk.check_self_contradiction(answer, spans, evidence, sentence_of=sentence_of)
121
+ findings += s_findings + c_findings
122
+ stats.update(s_stats)
123
+ stats.update(c_stats)
124
+ findings.sort(key=lambda f: (f.start, f.check))
125
+ # a number finding that a quote or a value-to-field finding replaced is no longer counted as a number
126
+ nums = [f for f in findings if f.check == "number"]
127
+ computed = sum(f.severity == "low" and numbers.COMPUTED_NOTE in f.message for f in nums)
128
+ stats["numbers_unsupported"] = sum(f.severity == "high" for f in nums)
129
+ stats["numbers_computed_in_answer"] = computed
130
+ stats["numbers_cut_short"] = sum(f.severity == "low" for f in nums) - computed
131
+ stats["sources"] = len(srcs)
132
+ stats["tool_calls"] = len(calls)
133
+ stats["tool_calls_failed"] = sum(c.error for c in calls)
134
+
135
+ n_num, n_quote, n_bind = stats["numbers_checked"], stats["quotes_checked"], stats["bindings_checked"]
136
+ parts = ([_n(n_num, "number")] if n_num else []) + ([_n(n_quote, "quote")] if n_quote else []) + \
137
+ ([_n(n_bind, "value-to-field binding")] if n_bind else [])
138
+ things = ", ".join(parts[:-1]) + (" and " if len(parts) > 1 else "") + (parts[-1] if parts else "")
139
+ problems = [(stats["numbers_unsupported"], "number", "not supported"),
140
+ (stats["numbers_cut_short"], "number", "cut short"),
141
+ (stats["numbers_computed_in_answer"], "number", "worked out in the answer, not by a tool"),
142
+ (stats["quotes_not_found"], "quote", "not found"),
143
+ (stats["quotes_wrong_source"], "quote", "in another source than the one cited"),
144
+ (stats["quotes_changed_slightly"], "quote", "changed slightly (small words, punctuation or spelling)"),
145
+ (stats["bindings_wrong"], "value", "attached to the wrong field"),
146
+ (stats["arguments_wrong"], "tool argument", "different from the question"),
147
+ (stats["tool_calls_failed_unreported"], "failed tool call", "reported as if it had worked"),
148
+ (stats["tool_claims_unbacked"], "claim", "about a tool that was not called"),
149
+ (stats["self_talk"], "passage", "of the model thinking aloud"),
150
+ (stats["self_contradictions"], "self-contradiction", "about what the sources say")]
151
+ if not has_evidence:
152
+ problems = problems[8:] # only the wording can be judged without evidence
153
+ found = [f"{_n(k, word)} {what}" for k, word, what in problems if k]
154
+ high = any(f.severity == "high" for f in findings)
155
+ if not has_evidence:
156
+ if high:
157
+ summary = "; ".join(found) + "." + (f" {things} not checked: no sources or question given." if things else "")
158
+ return Result(verdict="problems", findings=findings, summary=summary[0].upper() + summary[1:], stats=stats)
159
+ if n_num or n_quote:
160
+ return Result(verdict="unsure", findings=findings, stats=stats,
161
+ summary=f"{things} in the answer, but no sources or question to check against.")
162
+ return Result(verdict="ok", findings=findings, stats=stats, summary="No numbers or quotes to check.")
163
+ if not parts and not found:
164
+ return Result(verdict="ok", findings=findings, stats=stats, summary="No numbers or quotes to check.")
165
+ against = ([_n(len(srcs), "source")] if srcs else []) + ([_n(len(calls), "tool call")] if calls else []) + \
166
+ (["the question"] if has_q else [])
167
+ against_txt = ", ".join(against[:-1]) + (" and " if len(against) > 1 else "") + against[-1]
168
+ summary = (f"{things or 'The answer'} checked against {against_txt}: "
169
+ + ("; ".join(found) if found else "all supported") + ".")
170
+ return Result(verdict="problems" if high else "ok", findings=findings, summary=summary, stats=stats)
171
+
172
+
173
+ def _counts_text(call) -> str:
174
+ """How many items each list in the result has, as text the number check can use ("reviews: 3 items")."""
175
+ counts = binding.list_counts([call])
176
+ return ("\n(counts) " + "; ".join(f"{w}: {n} items" for w, n in counts)) if counts else ""
177
+
178
+
179
+ def check_messages(messages, sources=None, **options) -> Result:
180
+ """Check the final answer of a chat against its tool results, its question and any `sources`. The
181
+ chat can be OpenAI-style messages, Anthropic Messages API messages (tool_use and tool_result blocks)
182
+ or LangChain messages (objects or messages_to_dict dicts). Same options as check_answer()."""
183
+ return check_answer(None, sources, "", trace=messages, **options)
184
+
185
+
186
+ def _n(k: int, word: str) -> str:
187
+ return f"{k} {word}{'' if k == 1 else 's'}"