factguard 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- factguard-0.1.0/.github/workflows/ci.yml +30 -0
- factguard-0.1.0/.github/workflows/release.yml +34 -0
- factguard-0.1.0/.gitignore +10 -0
- factguard-0.1.0/CONTRIBUTING.md +42 -0
- factguard-0.1.0/LICENSE +21 -0
- factguard-0.1.0/PKG-INFO +175 -0
- factguard-0.1.0/README.md +153 -0
- factguard-0.1.0/examples/anthropic_analyst.py +66 -0
- factguard-0.1.0/pyproject.toml +47 -0
- factguard-0.1.0/src/factguard/__init__.py +29 -0
- factguard-0.1.0/src/factguard/__main__.py +5 -0
- factguard-0.1.0/src/factguard/cli.py +73 -0
- factguard-0.1.0/src/factguard/facts.py +211 -0
- factguard-0.1.0/src/factguard/guard.py +143 -0
- factguard-0.1.0/src/factguard/py.typed +0 -0
- factguard-0.1.0/src/factguard/sentences.py +56 -0
- factguard-0.1.0/src/factguard/testing.py +16 -0
- factguard-0.1.0/src/factguard/validate.py +252 -0
- factguard-0.1.0/tests/test_cli.py +44 -0
- factguard-0.1.0/tests/test_facts.py +103 -0
- factguard-0.1.0/tests/test_guard.py +111 -0
- factguard-0.1.0/tests/test_sentences.py +47 -0
- factguard-0.1.0/tests/test_validate.py +135 -0
- factguard-0.1.0/uv.lock +203 -0
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
name: ci
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
branches: [main]
|
|
6
|
+
pull_request:
|
|
7
|
+
|
|
8
|
+
permissions:
|
|
9
|
+
contents: read
|
|
10
|
+
|
|
11
|
+
jobs:
|
|
12
|
+
test:
|
|
13
|
+
strategy:
|
|
14
|
+
fail-fast: false # report every combination, not just the first failure
|
|
15
|
+
matrix:
|
|
16
|
+
os: [ubuntu-latest, macos-latest, windows-latest]
|
|
17
|
+
python: ["3.10", "3.11", "3.12", "3.13"]
|
|
18
|
+
runs-on: ${{ matrix.os }}
|
|
19
|
+
steps:
|
|
20
|
+
- uses: actions/checkout@v7
|
|
21
|
+
- uses: astral-sh/setup-uv@v7
|
|
22
|
+
with:
|
|
23
|
+
python-version: ${{ matrix.python }}
|
|
24
|
+
- run: uv sync
|
|
25
|
+
- run: uv run pytest
|
|
26
|
+
- name: lint
|
|
27
|
+
if: matrix.os == 'ubuntu-latest' && matrix.python == '3.13'
|
|
28
|
+
run: |
|
|
29
|
+
uv run ruff format --check .
|
|
30
|
+
uv run ruff check .
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
name: release
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
tags: ["v*"]
|
|
6
|
+
|
|
7
|
+
permissions:
|
|
8
|
+
contents: read
|
|
9
|
+
|
|
10
|
+
jobs:
|
|
11
|
+
build:
|
|
12
|
+
runs-on: ubuntu-latest
|
|
13
|
+
steps:
|
|
14
|
+
- uses: actions/checkout@v7
|
|
15
|
+
- uses: astral-sh/setup-uv@v7
|
|
16
|
+
- run: uv run pytest
|
|
17
|
+
- run: uv build
|
|
18
|
+
- uses: actions/upload-artifact@v7
|
|
19
|
+
with:
|
|
20
|
+
name: dist
|
|
21
|
+
path: dist/
|
|
22
|
+
|
|
23
|
+
pypi:
|
|
24
|
+
needs: build
|
|
25
|
+
runs-on: ubuntu-latest
|
|
26
|
+
environment: pypi
|
|
27
|
+
permissions:
|
|
28
|
+
id-token: write # PyPI trusted publishing, no stored token
|
|
29
|
+
steps:
|
|
30
|
+
- uses: actions/download-artifact@v8
|
|
31
|
+
with:
|
|
32
|
+
name: dist
|
|
33
|
+
path: dist/
|
|
34
|
+
- uses: pypa/gh-action-pypi-publish@release/v1
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
# Contributing to factguard
|
|
2
|
+
|
|
3
|
+
Thanks for your interest! factguard aims to stay small, dependency-free and
|
|
4
|
+
easy to read, so the best contributions are focused.
|
|
5
|
+
|
|
6
|
+
## Getting started
|
|
7
|
+
|
|
8
|
+
1. Fork the repo and create a branch from `main`.
|
|
9
|
+
2. Install [uv](https://docs.astral.sh/uv/) and set up the project:
|
|
10
|
+
```sh
|
|
11
|
+
uv sync
|
|
12
|
+
uv run pytest
|
|
13
|
+
```
|
|
14
|
+
3. Make your change with a test, then check formatting and lint:
|
|
15
|
+
```sh
|
|
16
|
+
uv run ruff format .
|
|
17
|
+
uv run ruff check .
|
|
18
|
+
```
|
|
19
|
+
4. Open a pull request describing what you changed and why.
|
|
20
|
+
|
|
21
|
+
CI runs the tests on Python 3.10 to 3.13 on Linux, macOS and Windows, so
|
|
22
|
+
please make sure it stays green.
|
|
23
|
+
|
|
24
|
+
## Good first contributions
|
|
25
|
+
|
|
26
|
+
- **False positives and misses.** A sentence that was flagged but shouldn't be,
|
|
27
|
+
or let through but shouldn't be. Add it as a test case first.
|
|
28
|
+
- **Date formats.** More ways of writing dates and periods that should be
|
|
29
|
+
checked against the analysis window.
|
|
30
|
+
- **Formatting.** Locale-aware number formats, such as lakh and crore grouping.
|
|
31
|
+
- **Examples.** Short examples for other LLM SDKs.
|
|
32
|
+
|
|
33
|
+
For larger changes (new rule kinds, API changes, any new dependency), please
|
|
34
|
+
open an issue first so we can agree on the approach.
|
|
35
|
+
|
|
36
|
+
## Code style
|
|
37
|
+
|
|
38
|
+
- Python 3.10+, standard library only.
|
|
39
|
+
- Readable over clever. Prefer small functions and plain loops.
|
|
40
|
+
- Strict by default: a new allowance should be narrow and, where in doubt,
|
|
41
|
+
opt-in.
|
|
42
|
+
- Write brief comments explaining *why* a rule exists.
|
factguard-0.1.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Shivam Singh
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
factguard-0.1.0/PKG-INFO
ADDED
|
@@ -0,0 +1,175 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: factguard
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Keep LLM-written text free of numbers your code didn't compute: fact placeholders, a strict validator, one retry, then drop the sentence
|
|
5
|
+
Project-URL: Homepage, https://github.com/shivamsingh262/factguard
|
|
6
|
+
Project-URL: Issues, https://github.com/shivamsingh262/factguard/issues
|
|
7
|
+
Project-URL: Design notes, https://shivamsingh262.github.io/notes/code-computes-llm-interprets/
|
|
8
|
+
Author: Shivam Singh
|
|
9
|
+
License-Expression: MIT
|
|
10
|
+
License-File: LICENSE
|
|
11
|
+
Keywords: ai-agents,analytics,grounding,guardrails,hallucination,llm
|
|
12
|
+
Classifier: Development Status :: 3 - Alpha
|
|
13
|
+
Classifier: Intended Audience :: Developers
|
|
14
|
+
Classifier: Operating System :: OS Independent
|
|
15
|
+
Classifier: Programming Language :: Python :: 3
|
|
16
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
17
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
18
|
+
Classifier: Topic :: Software Development :: Libraries
|
|
19
|
+
Classifier: Typing :: Typed
|
|
20
|
+
Requires-Python: >=3.10
|
|
21
|
+
Description-Content-Type: text/markdown
|
|
22
|
+
|
|
23
|
+
# factguard
|
|
24
|
+
|
|
25
|
+
[](https://github.com/shivamsingh262/factguard/actions/workflows/ci.yml)
|
|
26
|
+
[](LICENSE)
|
|
27
|
+

|
|
28
|
+

|
|
29
|
+
[](CONTRIBUTING.md)
|
|
30
|
+
|
|
31
|
+
Keep LLM-written text free of numbers your code didn't compute.
|
|
32
|
+
|
|
33
|
+
If an LLM writes a wrong number confidently, users stop trusting every other
|
|
34
|
+
number on the screen. factguard enforces one rule: **code computes every
|
|
35
|
+
number, the LLM only interprets.**
|
|
36
|
+
|
|
37
|
+
- Your code registers each number as a **fact**: a key, a value, a format
|
|
38
|
+
(money, percent, count, ratio...) and where it came from.
|
|
39
|
+
- The LLM cites facts as placeholders like `{{fact:spend}}` and never types the
|
|
40
|
+
digits itself.
|
|
41
|
+
- A **validator** rejects any literal number, unknown fact key, or date or
|
|
42
|
+
duration outside the analysis window.
|
|
43
|
+
- On failure the model gets **one retry** with a message naming exactly what
|
|
44
|
+
broke. If the retry fails too, the offending **sentences are dropped**. A
|
|
45
|
+
shorter answer is better than a wrong one.
|
|
46
|
+
|
|
47
|
+
It works with any LLM and any SDK, and has no dependencies. The design comes
|
|
48
|
+
from a production AI analyst for marketing teams; the write-up is
|
|
49
|
+
[Stopping an LLM from making up numbers in an AI analyst](https://shivamsingh262.github.io/notes/code-computes-llm-interprets/).
|
|
50
|
+
|
|
51
|
+
## Install
|
|
52
|
+
|
|
53
|
+
```sh
|
|
54
|
+
pip install git+https://github.com/shivamsingh262/factguard
|
|
55
|
+
```
|
|
56
|
+
|
|
57
|
+
## Quick start
|
|
58
|
+
|
|
59
|
+
```python
|
|
60
|
+
from datetime import date
|
|
61
|
+
from factguard import Facts, guard
|
|
62
|
+
|
|
63
|
+
facts = Facts(currency="INR")
|
|
64
|
+
facts.add("spend", 61_240, "money", label="Ad spend this week")
|
|
65
|
+
facts.add("roas", 1.8, "ratio", label="ROAS this week")
|
|
66
|
+
facts.add("roas_prev", 2.4, "ratio", label="ROAS last week")
|
|
67
|
+
|
|
68
|
+
system_prompt = facts.prompt() # the rules, plus "- {{fact:spend}} = INR 61.2K (...)"
|
|
69
|
+
|
|
70
|
+
def generate(retry):
|
|
71
|
+
# Call your LLM here. On a retry, show it retry.draft and retry.feedback.
|
|
72
|
+
...
|
|
73
|
+
|
|
74
|
+
week = (date(2026, 9, 22), date(2026, 9, 28))
|
|
75
|
+
result = guard(generate, facts, windows=[week], user_text="Why did ROAS drop?")
|
|
76
|
+
|
|
77
|
+
result.text # "ROAS fell from 2.4x to 1.8x while spend held at INR 61.2K."
|
|
78
|
+
result.dropped # sentences removed because they still broke the rules
|
|
79
|
+
result.attempts # 1 or 2
|
|
80
|
+
```
|
|
81
|
+
|
|
82
|
+
[`examples/anthropic_analyst.py`](examples/anthropic_analyst.py) is a complete
|
|
83
|
+
example using Claude.
|
|
84
|
+
|
|
85
|
+
## What counts as a violation
|
|
86
|
+
|
|
87
|
+
| Kind | Example | Why it fails |
|
|
88
|
+
|---|---|---|
|
|
89
|
+
| `literal_number` | `CPA is up 40%` | The model typed a number. Cite a fact or say it without the figure. |
|
|
90
|
+
| `unknown_fact` | `{{fact:f_9999.roas}}` | No such fact was computed. |
|
|
91
|
+
| `malformed_placeholder` | `{{spend}}` | Placeholders must be exactly `{{fact:key}}`. |
|
|
92
|
+
| `date` | `since Oct 3` | Dates, months and years must fall inside a window you passed. |
|
|
93
|
+
| `duration` | `over the last 45 days` | Durations must match the length of a window you passed. |
|
|
94
|
+
| `number_word` | `spend nearly doubled` | Only with `number_words=True`. |
|
|
95
|
+
|
|
96
|
+
Allowed without a fact:
|
|
97
|
+
|
|
98
|
+
- numbered list markers (`1.`, `2)`)
|
|
99
|
+
- attribution labels (`7-day click`, `1-day view`)
|
|
100
|
+
- names like `GA4`, `B2B`, `3rd-party` and `24/7`
|
|
101
|
+
- numbers the user typed (`user_text`), so "campaigns with ROAS below 2" can
|
|
102
|
+
repeat the 2
|
|
103
|
+
- your own patterns: `allow=[r"SKU-\d+"]`
|
|
104
|
+
|
|
105
|
+
Without `windows`, every date and duration fails. That is deliberate: the
|
|
106
|
+
model shouldn't do date arithmetic. Resolve "last month" in code and pass the
|
|
107
|
+
window you actually queried. Pass both windows when you compare two periods.
|
|
108
|
+
|
|
109
|
+
## The pieces
|
|
110
|
+
|
|
111
|
+
**`Facts`** holds the facts for one answer.
|
|
112
|
+
|
|
113
|
+
- `facts.add(key, value, kind, label=..., currency=..., decimals=..., source=..., step=...)`
|
|
114
|
+
- `facts.prompt()` returns the citation rules and fact list for your system prompt.
|
|
115
|
+
- `facts.render(text)` replaces placeholders with formatted values. It also strips a currency the model typed in
|
|
116
|
+
front of a money placeholder (`INR {{fact:spend}}`), since the fact brings its own.
|
|
117
|
+
- `Facts.from_records(json_data)` loads facts from JSON.
|
|
118
|
+
|
|
119
|
+
Formats: `money` renders as `INR 61.2K` or `$1.2M` (pass `Formatter(compact=False)` for `61,240`). `percent` takes
|
|
120
|
+
fractions, so `0.034` renders as `3.4%`. `count` renders as `12,480`, `ratio` as `2.3x`, and `text` as-is. To use
|
|
121
|
+
your own formatting, pass any `Callable[[Fact], str]` as `formatter=`.
|
|
122
|
+
|
|
123
|
+
**`validate(text, facts, **rules)`** returns a `Report` with `.ok`, `.violations` (kind, token, start, end) and
|
|
124
|
+
`.feedback`, a retry message that names each offending token.
|
|
125
|
+
|
|
126
|
+
**`guard(generate, facts, retries=1, **rules)`** runs the whole loop. `generate(None)` should return a first draft,
|
|
127
|
+
and `generate(retry)` a corrected one. Use `aguard` for async code.
|
|
128
|
+
|
|
129
|
+
**`StreamGuard(facts, **rules)`** checks streamed text one sentence at a time. A sentence already on screen can't
|
|
130
|
+
be rewritten, so a failing sentence is dropped instead of retried:
|
|
131
|
+
|
|
132
|
+
```python
|
|
133
|
+
sg = StreamGuard(facts, windows=[week])
|
|
134
|
+
for chunk in llm_stream:
|
|
135
|
+
for sentence in sg.feed(chunk):
|
|
136
|
+
ui.append(sentence)
|
|
137
|
+
for sentence in sg.close():
|
|
138
|
+
ui.append(sentence)
|
|
139
|
+
```
|
|
140
|
+
|
|
141
|
+
**Evals.** Check every recorded answer for zero ungrounded numbers, not just for the right answer:
|
|
142
|
+
|
|
143
|
+
```python
|
|
144
|
+
from factguard.testing import assert_grounded
|
|
145
|
+
|
|
146
|
+
def test_recorded_answer():
|
|
147
|
+
assert_grounded(answer, facts, windows=[week])
|
|
148
|
+
```
|
|
149
|
+
|
|
150
|
+
Or from the shell or CI:
|
|
151
|
+
|
|
152
|
+
```sh
|
|
153
|
+
factguard check answer.txt --facts facts.json --window 2026-09-22:2026-09-28
|
|
154
|
+
factguard render answer.txt --facts facts.json
|
|
155
|
+
```
|
|
156
|
+
|
|
157
|
+
## Trade-offs
|
|
158
|
+
|
|
159
|
+
The prose gets stiffer. Without "nearly double" or "about a third", the model
|
|
160
|
+
writes more literal sentences. Some sentences get dropped, and a retry adds
|
|
161
|
+
latency. If users have to trust the numbers before the interpretation is worth
|
|
162
|
+
anything, that trade is worth it.
|
|
163
|
+
|
|
164
|
+
Placeholders don't stop the model from citing the *wrong* fact, so keep fact
|
|
165
|
+
labels clear and test with recorded questions. And `number_words` is a
|
|
166
|
+
heuristic: it is off by default because words like "half" are often harmless.
|
|
167
|
+
|
|
168
|
+
## Contributing
|
|
169
|
+
|
|
170
|
+
See [CONTRIBUTING.md](CONTRIBUTING.md). Bug reports with a sentence that was
|
|
171
|
+
wrongly flagged or wrongly let through are especially useful.
|
|
172
|
+
|
|
173
|
+
## License
|
|
174
|
+
|
|
175
|
+
[MIT](LICENSE)
|
|
@@ -0,0 +1,153 @@
|
|
|
1
|
+
# factguard
|
|
2
|
+
|
|
3
|
+
[](https://github.com/shivamsingh262/factguard/actions/workflows/ci.yml)
|
|
4
|
+
[](LICENSE)
|
|
5
|
+

|
|
6
|
+

|
|
7
|
+
[](CONTRIBUTING.md)
|
|
8
|
+
|
|
9
|
+
Keep LLM-written text free of numbers your code didn't compute.
|
|
10
|
+
|
|
11
|
+
If an LLM writes a wrong number confidently, users stop trusting every other
|
|
12
|
+
number on the screen. factguard enforces one rule: **code computes every
|
|
13
|
+
number, the LLM only interprets.**
|
|
14
|
+
|
|
15
|
+
- Your code registers each number as a **fact**: a key, a value, a format
|
|
16
|
+
(money, percent, count, ratio...) and where it came from.
|
|
17
|
+
- The LLM cites facts as placeholders like `{{fact:spend}}` and never types the
|
|
18
|
+
digits itself.
|
|
19
|
+
- A **validator** rejects any literal number, unknown fact key, or date or
|
|
20
|
+
duration outside the analysis window.
|
|
21
|
+
- On failure the model gets **one retry** with a message naming exactly what
|
|
22
|
+
broke. If the retry fails too, the offending **sentences are dropped**. A
|
|
23
|
+
shorter answer is better than a wrong one.
|
|
24
|
+
|
|
25
|
+
It works with any LLM and any SDK, and has no dependencies. The design comes
|
|
26
|
+
from a production AI analyst for marketing teams; the write-up is
|
|
27
|
+
[Stopping an LLM from making up numbers in an AI analyst](https://shivamsingh262.github.io/notes/code-computes-llm-interprets/).
|
|
28
|
+
|
|
29
|
+
## Install
|
|
30
|
+
|
|
31
|
+
```sh
|
|
32
|
+
pip install git+https://github.com/shivamsingh262/factguard
|
|
33
|
+
```
|
|
34
|
+
|
|
35
|
+
## Quick start
|
|
36
|
+
|
|
37
|
+
```python
|
|
38
|
+
from datetime import date
|
|
39
|
+
from factguard import Facts, guard
|
|
40
|
+
|
|
41
|
+
facts = Facts(currency="INR")
|
|
42
|
+
facts.add("spend", 61_240, "money", label="Ad spend this week")
|
|
43
|
+
facts.add("roas", 1.8, "ratio", label="ROAS this week")
|
|
44
|
+
facts.add("roas_prev", 2.4, "ratio", label="ROAS last week")
|
|
45
|
+
|
|
46
|
+
system_prompt = facts.prompt() # the rules, plus "- {{fact:spend}} = INR 61.2K (...)"
|
|
47
|
+
|
|
48
|
+
def generate(retry):
|
|
49
|
+
# Call your LLM here. On a retry, show it retry.draft and retry.feedback.
|
|
50
|
+
...
|
|
51
|
+
|
|
52
|
+
week = (date(2026, 9, 22), date(2026, 9, 28))
|
|
53
|
+
result = guard(generate, facts, windows=[week], user_text="Why did ROAS drop?")
|
|
54
|
+
|
|
55
|
+
result.text # "ROAS fell from 2.4x to 1.8x while spend held at INR 61.2K."
|
|
56
|
+
result.dropped # sentences removed because they still broke the rules
|
|
57
|
+
result.attempts # 1 or 2
|
|
58
|
+
```
|
|
59
|
+
|
|
60
|
+
[`examples/anthropic_analyst.py`](examples/anthropic_analyst.py) is a complete
|
|
61
|
+
example using Claude.
|
|
62
|
+
|
|
63
|
+
## What counts as a violation
|
|
64
|
+
|
|
65
|
+
| Kind | Example | Why it fails |
|
|
66
|
+
|---|---|---|
|
|
67
|
+
| `literal_number` | `CPA is up 40%` | The model typed a number. Cite a fact or say it without the figure. |
|
|
68
|
+
| `unknown_fact` | `{{fact:f_9999.roas}}` | No such fact was computed. |
|
|
69
|
+
| `malformed_placeholder` | `{{spend}}` | Placeholders must be exactly `{{fact:key}}`. |
|
|
70
|
+
| `date` | `since Oct 3` | Dates, months and years must fall inside a window you passed. |
|
|
71
|
+
| `duration` | `over the last 45 days` | Durations must match the length of a window you passed. |
|
|
72
|
+
| `number_word` | `spend nearly doubled` | Only with `number_words=True`. |
|
|
73
|
+
|
|
74
|
+
Allowed without a fact:
|
|
75
|
+
|
|
76
|
+
- numbered list markers (`1.`, `2)`)
|
|
77
|
+
- attribution labels (`7-day click`, `1-day view`)
|
|
78
|
+
- names like `GA4`, `B2B`, `3rd-party` and `24/7`
|
|
79
|
+
- numbers the user typed (`user_text`), so "campaigns with ROAS below 2" can
|
|
80
|
+
repeat the 2
|
|
81
|
+
- your own patterns: `allow=[r"SKU-\d+"]`
|
|
82
|
+
|
|
83
|
+
Without `windows`, every date and duration fails. That is deliberate: the
|
|
84
|
+
model shouldn't do date arithmetic. Resolve "last month" in code and pass the
|
|
85
|
+
window you actually queried. Pass both windows when you compare two periods.
|
|
86
|
+
|
|
87
|
+
## The pieces
|
|
88
|
+
|
|
89
|
+
**`Facts`** holds the facts for one answer.
|
|
90
|
+
|
|
91
|
+
- `facts.add(key, value, kind, label=..., currency=..., decimals=..., source=..., step=...)`
|
|
92
|
+
- `facts.prompt()` returns the citation rules and fact list for your system prompt.
|
|
93
|
+
- `facts.render(text)` replaces placeholders with formatted values. It also strips a currency the model typed in
|
|
94
|
+
front of a money placeholder (`INR {{fact:spend}}`), since the fact brings its own.
|
|
95
|
+
- `Facts.from_records(json_data)` loads facts from JSON.
|
|
96
|
+
|
|
97
|
+
Formats: `money` renders as `INR 61.2K` or `$1.2M` (pass `Formatter(compact=False)` for `61,240`). `percent` takes
|
|
98
|
+
fractions, so `0.034` renders as `3.4%`. `count` renders as `12,480`, `ratio` as `2.3x`, and `text` as-is. To use
|
|
99
|
+
your own formatting, pass any `Callable[[Fact], str]` as `formatter=`.
|
|
100
|
+
|
|
101
|
+
**`validate(text, facts, **rules)`** returns a `Report` with `.ok`, `.violations` (kind, token, start, end) and
|
|
102
|
+
`.feedback`, a retry message that names each offending token.
|
|
103
|
+
|
|
104
|
+
**`guard(generate, facts, retries=1, **rules)`** runs the whole loop. `generate(None)` should return a first draft,
|
|
105
|
+
and `generate(retry)` a corrected one. Use `aguard` for async code.
|
|
106
|
+
|
|
107
|
+
**`StreamGuard(facts, **rules)`** checks streamed text one sentence at a time. A sentence already on screen can't
|
|
108
|
+
be rewritten, so a failing sentence is dropped instead of retried:
|
|
109
|
+
|
|
110
|
+
```python
|
|
111
|
+
sg = StreamGuard(facts, windows=[week])
|
|
112
|
+
for chunk in llm_stream:
|
|
113
|
+
for sentence in sg.feed(chunk):
|
|
114
|
+
ui.append(sentence)
|
|
115
|
+
for sentence in sg.close():
|
|
116
|
+
ui.append(sentence)
|
|
117
|
+
```
|
|
118
|
+
|
|
119
|
+
**Evals.** Check every recorded answer for zero ungrounded numbers, not just for the right answer:
|
|
120
|
+
|
|
121
|
+
```python
|
|
122
|
+
from factguard.testing import assert_grounded
|
|
123
|
+
|
|
124
|
+
def test_recorded_answer():
|
|
125
|
+
assert_grounded(answer, facts, windows=[week])
|
|
126
|
+
```
|
|
127
|
+
|
|
128
|
+
Or from the shell or CI:
|
|
129
|
+
|
|
130
|
+
```sh
|
|
131
|
+
factguard check answer.txt --facts facts.json --window 2026-09-22:2026-09-28
|
|
132
|
+
factguard render answer.txt --facts facts.json
|
|
133
|
+
```
|
|
134
|
+
|
|
135
|
+
## Trade-offs
|
|
136
|
+
|
|
137
|
+
The prose gets stiffer. Without "nearly double" or "about a third", the model
|
|
138
|
+
writes more literal sentences. Some sentences get dropped, and a retry adds
|
|
139
|
+
latency. If users have to trust the numbers before the interpretation is worth
|
|
140
|
+
anything, that trade is worth it.
|
|
141
|
+
|
|
142
|
+
Placeholders don't stop the model from citing the *wrong* fact, so keep fact
|
|
143
|
+
labels clear and test with recorded questions. And `number_words` is a
|
|
144
|
+
heuristic: it is off by default because words like "half" are often harmless.
|
|
145
|
+
|
|
146
|
+
## Contributing
|
|
147
|
+
|
|
148
|
+
See [CONTRIBUTING.md](CONTRIBUTING.md). Bug reports with a sentence that was
|
|
149
|
+
wrongly flagged or wrongly let through are especially useful.
|
|
150
|
+
|
|
151
|
+
## License
|
|
152
|
+
|
|
153
|
+
[MIT](LICENSE)
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
"""Ask Claude why ROAS dropped, with every number in the answer guarded.
|
|
2
|
+
|
|
3
|
+
pip install factguard anthropic
|
|
4
|
+
export ANTHROPIC_API_KEY=...
|
|
5
|
+
python examples/anthropic_analyst.py
|
|
6
|
+
|
|
7
|
+
The numbers below are made up. In a real system they come from your own
|
|
8
|
+
queries; the point is that code computes them and the model only cites them.
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
from datetime import date
|
|
12
|
+
|
|
13
|
+
import anthropic
|
|
14
|
+
|
|
15
|
+
from factguard import Facts, Retry, guard
|
|
16
|
+
|
|
17
|
+
this_week = (date(2026, 9, 22), date(2026, 9, 28))
|
|
18
|
+
last_week = (date(2026, 9, 15), date(2026, 9, 21))
|
|
19
|
+
|
|
20
|
+
facts = Facts(currency="INR")
|
|
21
|
+
facts.add("spend", 61_240, "money", label="Ad spend this week", source="meta_ads")
|
|
22
|
+
facts.add("spend_prev", 59_800, "money", label="Ad spend last week", source="meta_ads")
|
|
23
|
+
facts.add("roas", 1.8, "ratio", label="ROAS this week", source="shopify + meta_ads")
|
|
24
|
+
facts.add("roas_prev", 2.4, "ratio", label="ROAS last week", source="shopify + meta_ads")
|
|
25
|
+
facts.add("ctr", 0.0121, "percent", decimals=2, label="CTR this week", source="meta_ads")
|
|
26
|
+
facts.add("ctr_prev", 0.0184, "percent", decimals=2, label="CTR last week", source="meta_ads")
|
|
27
|
+
facts.add("top_ad", "Summer UGC v3", "text", label="Ad set with the biggest CTR drop", source="meta_ads")
|
|
28
|
+
|
|
29
|
+
question = "Why did ROAS drop last week?"
|
|
30
|
+
system = (
|
|
31
|
+
"You are a marketing analyst. Answer in a short paragraph or a few bullets.\n"
|
|
32
|
+
f"This week is {this_week[0]:%b %d} to {this_week[1]:%b %d}; "
|
|
33
|
+
f"it is compared with {last_week[0]:%b %d} to {last_week[1]:%b %d}.\n\n" + facts.prompt()
|
|
34
|
+
)
|
|
35
|
+
|
|
36
|
+
client = anthropic.Anthropic()
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def generate(retry: Retry | None) -> str:
|
|
40
|
+
messages = [{"role": "user", "content": question}]
|
|
41
|
+
if retry:
|
|
42
|
+
# Show the model its own draft and exactly what broke the rules.
|
|
43
|
+
messages += [
|
|
44
|
+
{"role": "assistant", "content": retry.draft},
|
|
45
|
+
{"role": "user", "content": retry.feedback},
|
|
46
|
+
]
|
|
47
|
+
response = client.beta.messages.create(
|
|
48
|
+
model="claude-opus-5-5",
|
|
49
|
+
max_tokens=16000,
|
|
50
|
+
system=system,
|
|
51
|
+
messages=messages,
|
|
52
|
+
# If the model declines, the API reruns the request on a fallback model.
|
|
53
|
+
betas=["server-side-fallback-2026-07-01"],
|
|
54
|
+
fallbacks="default",
|
|
55
|
+
)
|
|
56
|
+
if response.stop_reason == "refusal":
|
|
57
|
+
raise RuntimeError("the model declined to answer")
|
|
58
|
+
return "".join(block.text for block in response.content if block.type == "text")
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
result = guard(generate, facts, windows=[this_week, last_week], user_text=question)
|
|
62
|
+
|
|
63
|
+
print(result.text)
|
|
64
|
+
print(f"\n[{result.attempts} attempt(s)]")
|
|
65
|
+
for sentence in result.dropped:
|
|
66
|
+
print(f"[dropped] {sentence}")
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["hatchling>=1.25"]
|
|
3
|
+
build-backend = "hatchling.build"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "factguard"
|
|
7
|
+
version = "0.1.0"
|
|
8
|
+
description = "Keep LLM-written text free of numbers your code didn't compute: fact placeholders, a strict validator, one retry, then drop the sentence"
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
license = "MIT"
|
|
11
|
+
license-files = ["LICENSE"]
|
|
12
|
+
authors = [{ name = "Shivam Singh" }]
|
|
13
|
+
requires-python = ">=3.10"
|
|
14
|
+
dependencies = []
|
|
15
|
+
keywords = ["llm", "hallucination", "grounding", "ai-agents", "guardrails", "analytics"]
|
|
16
|
+
classifiers = [
|
|
17
|
+
"Development Status :: 3 - Alpha",
|
|
18
|
+
"Intended Audience :: Developers",
|
|
19
|
+
"Operating System :: OS Independent",
|
|
20
|
+
"Programming Language :: Python :: 3",
|
|
21
|
+
"Programming Language :: Python :: 3 :: Only",
|
|
22
|
+
"Topic :: Scientific/Engineering :: Artificial Intelligence",
|
|
23
|
+
"Topic :: Software Development :: Libraries",
|
|
24
|
+
"Typing :: Typed",
|
|
25
|
+
]
|
|
26
|
+
|
|
27
|
+
[project.urls]
|
|
28
|
+
Homepage = "https://github.com/shivamsingh262/factguard"
|
|
29
|
+
Issues = "https://github.com/shivamsingh262/factguard/issues"
|
|
30
|
+
"Design notes" = "https://shivamsingh262.github.io/notes/code-computes-llm-interprets/"
|
|
31
|
+
|
|
32
|
+
[project.scripts]
|
|
33
|
+
factguard = "factguard.cli:main"
|
|
34
|
+
|
|
35
|
+
[dependency-groups]
|
|
36
|
+
dev = ["pytest>=8", "ruff>=0.6"]
|
|
37
|
+
|
|
38
|
+
[tool.ruff]
|
|
39
|
+
line-length = 120
|
|
40
|
+
target-version = "py310"
|
|
41
|
+
extend-exclude = ["*.md"]
|
|
42
|
+
|
|
43
|
+
[tool.ruff.lint]
|
|
44
|
+
select = ["E", "F", "I", "UP", "B"]
|
|
45
|
+
|
|
46
|
+
[tool.pytest.ini_options]
|
|
47
|
+
testpaths = ["tests"]
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
"""Code computes every number, the LLM only interprets.
|
|
2
|
+
|
|
3
|
+
factguard keeps LLM-written text free of numbers your code didn't compute.
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
from .facts import RULES, Fact, Facts, Formatter
|
|
7
|
+
from .guard import Result, Retry, StreamGuard, aguard, guard
|
|
8
|
+
from .sentences import drop_sentences, split_sentences
|
|
9
|
+
from .validate import Report, Violation, Window, validate
|
|
10
|
+
|
|
11
|
+
__version__ = "0.1.0"
|
|
12
|
+
|
|
13
|
+
__all__ = [
|
|
14
|
+
"RULES",
|
|
15
|
+
"Fact",
|
|
16
|
+
"Facts",
|
|
17
|
+
"Formatter",
|
|
18
|
+
"Report",
|
|
19
|
+
"Result",
|
|
20
|
+
"Retry",
|
|
21
|
+
"StreamGuard",
|
|
22
|
+
"Violation",
|
|
23
|
+
"Window",
|
|
24
|
+
"aguard",
|
|
25
|
+
"drop_sentences",
|
|
26
|
+
"guard",
|
|
27
|
+
"split_sentences",
|
|
28
|
+
"validate",
|
|
29
|
+
]
|