verbatim-citation-gate 0.3.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- verbatim_citation_gate-0.3.2/.github/release.yml +33 -0
- verbatim_citation_gate-0.3.2/.github/workflows/publish.yml +50 -0
- verbatim_citation_gate-0.3.2/.github/workflows/tests.yml +101 -0
- verbatim_citation_gate-0.3.2/.gitignore +7 -0
- verbatim_citation_gate-0.3.2/AGENTS.md +67 -0
- verbatim_citation_gate-0.3.2/CHANGELOG.md +87 -0
- verbatim_citation_gate-0.3.2/CITATION.cff +27 -0
- verbatim_citation_gate-0.3.2/FOR-ROBOTS.md +90 -0
- verbatim_citation_gate-0.3.2/GOVERNANCE.md +44 -0
- verbatim_citation_gate-0.3.2/LICENSE +21 -0
- verbatim_citation_gate-0.3.2/PKG-INFO +367 -0
- verbatim_citation_gate-0.3.2/README.md +316 -0
- verbatim_citation_gate-0.3.2/SECURITY.md +42 -0
- verbatim_citation_gate-0.3.2/bench/README.md +91 -0
- verbatim_citation_gate-0.3.2/bench/bench_gate.py +274 -0
- verbatim_citation_gate-0.3.2/bench/build_corpus.py +67 -0
- verbatim_citation_gate-0.3.2/bench/corpus.json +297 -0
- verbatim_citation_gate-0.3.2/bench/hard_cases.py +135 -0
- verbatim_citation_gate-0.3.2/bench/results.json +393 -0
- verbatim_citation_gate-0.3.2/docs/.nojekyll +0 -0
- verbatim_citation_gate-0.3.2/docs/demo.gif +0 -0
- verbatim_citation_gate-0.3.2/docs/index.html +263 -0
- verbatim_citation_gate-0.3.2/docs/make_demo_gif.py +127 -0
- verbatim_citation_gate-0.3.2/examples/demo.py +59 -0
- verbatim_citation_gate-0.3.2/examples/quickstart.py +49 -0
- verbatim_citation_gate-0.3.2/pyproject.toml +52 -0
- verbatim_citation_gate-0.3.2/src/verbatim_citation_gate/__init__.py +39 -0
- verbatim_citation_gate-0.3.2/src/verbatim_citation_gate/gate.py +331 -0
- verbatim_citation_gate-0.3.2/src/verbatim_citation_gate/judge.py +152 -0
- verbatim_citation_gate-0.3.2/src/verbatim_citation_gate/py.typed +0 -0
- verbatim_citation_gate-0.3.2/tests/test_gate.py +412 -0
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
# Categories for auto-generated release notes.
|
|
2
|
+
#
|
|
3
|
+
# Unlike CONTRIBUTING / issue templates / the PR template, this file is NOT inherited
|
|
4
|
+
# from the organisation `.github` repository — GitHub reads it per repository only.
|
|
5
|
+
# So this is a required local file, not a copy shadowing a canon somewhere else.
|
|
6
|
+
#
|
|
7
|
+
# Docs: https://docs.github.com/en/repositories/releasing-projects-on-github/automatically-generated-release-notes
|
|
8
|
+
|
|
9
|
+
changelog:
|
|
10
|
+
exclude:
|
|
11
|
+
labels:
|
|
12
|
+
- duplicate
|
|
13
|
+
- invalid
|
|
14
|
+
- wontfix
|
|
15
|
+
- question
|
|
16
|
+
categories:
|
|
17
|
+
- title: Fixed — it used to behave wrong
|
|
18
|
+
labels:
|
|
19
|
+
- bug
|
|
20
|
+
- title: New
|
|
21
|
+
labels:
|
|
22
|
+
- enhancement
|
|
23
|
+
- title: Docs and onboarding
|
|
24
|
+
labels:
|
|
25
|
+
- documentation
|
|
26
|
+
- title: First contribution here
|
|
27
|
+
labels:
|
|
28
|
+
- good first issue
|
|
29
|
+
# Catch-all. Nothing merged is allowed to fall out of the notes silently:
|
|
30
|
+
# an unlabelled PR lands here rather than disappearing.
|
|
31
|
+
- title: Other changes
|
|
32
|
+
labels:
|
|
33
|
+
- "*"
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
# Publish to PyPI on every published GitHub release.
|
|
2
|
+
#
|
|
3
|
+
# Uses PyPI Trusted Publishing (OIDC): no API token is stored anywhere, not in a
|
|
4
|
+
# repository secret and not on a laptop. PyPI verifies the workflow's identity
|
|
5
|
+
# directly. Set it up once at
|
|
6
|
+
# https://pypi.org/manage/account/publishing/
|
|
7
|
+
# with owner "tonydzi", repository "verbatim-citation-gate", workflow "publish.yml",
|
|
8
|
+
# environment "pypi".
|
|
9
|
+
name: publish
|
|
10
|
+
|
|
11
|
+
on:
|
|
12
|
+
release:
|
|
13
|
+
types: [published]
|
|
14
|
+
workflow_dispatch:
|
|
15
|
+
|
|
16
|
+
permissions:
|
|
17
|
+
contents: read
|
|
18
|
+
|
|
19
|
+
jobs:
|
|
20
|
+
build:
|
|
21
|
+
runs-on: ubuntu-latest
|
|
22
|
+
steps:
|
|
23
|
+
- uses: actions/checkout@v4
|
|
24
|
+
- uses: actions/setup-python@v5
|
|
25
|
+
with:
|
|
26
|
+
python-version: "3.12"
|
|
27
|
+
- name: Build sdist and wheel
|
|
28
|
+
run: |
|
|
29
|
+
python -m pip install --upgrade pip build twine
|
|
30
|
+
python -m build
|
|
31
|
+
python -m twine check --strict dist/*
|
|
32
|
+
- uses: actions/upload-artifact@v4
|
|
33
|
+
with:
|
|
34
|
+
name: dist
|
|
35
|
+
path: dist/
|
|
36
|
+
|
|
37
|
+
publish:
|
|
38
|
+
needs: build
|
|
39
|
+
runs-on: ubuntu-latest
|
|
40
|
+
# Skipped until the PyPI Trusted Publisher exists (repo variable PYPI_READY=true), so a release is not a red job.
|
|
41
|
+
if: ${{ vars.PYPI_READY == 'true' }}
|
|
42
|
+
environment: pypi
|
|
43
|
+
permissions:
|
|
44
|
+
id-token: write # the whole point: OIDC instead of a stored token
|
|
45
|
+
steps:
|
|
46
|
+
- uses: actions/download-artifact@v4
|
|
47
|
+
with:
|
|
48
|
+
name: dist
|
|
49
|
+
path: dist/
|
|
50
|
+
- uses: pypa/gh-action-pypi-publish@release/v1
|
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
name: tests
|
|
2
|
+
|
|
3
|
+
# The suite existed since day one and nothing ran it on a schedule.
|
|
4
|
+
# A test nobody runs is not a test — so it runs on every push and weekly.
|
|
5
|
+
on:
|
|
6
|
+
push:
|
|
7
|
+
pull_request:
|
|
8
|
+
workflow_dispatch:
|
|
9
|
+
schedule:
|
|
10
|
+
- cron: "17 5 * * 1" # weekly, so rot surfaces without a push
|
|
11
|
+
|
|
12
|
+
jobs:
|
|
13
|
+
tests:
|
|
14
|
+
runs-on: ${{ matrix.os }}
|
|
15
|
+
strategy:
|
|
16
|
+
fail-fast: false
|
|
17
|
+
matrix:
|
|
18
|
+
# ubuntu-latest became 26.04 in Nov 2026 and setup-python has no 3.9 for it.
|
|
19
|
+
# 3.9 is the declared floor, so pin the image instead of dropping the floor.
|
|
20
|
+
os: [ubuntu-24.04, macos-latest, windows-latest]
|
|
21
|
+
python: ["3.9", "3.11", "3.13"]
|
|
22
|
+
steps:
|
|
23
|
+
- uses: actions/checkout@v4
|
|
24
|
+
- uses: actions/setup-python@v5
|
|
25
|
+
with:
|
|
26
|
+
python-version: ${{ matrix.python }}
|
|
27
|
+
- name: install with the test extra
|
|
28
|
+
run: pip install -e ".[test]"
|
|
29
|
+
- name: gate + judge tests
|
|
30
|
+
run: pytest -q
|
|
31
|
+
|
|
32
|
+
package:
|
|
33
|
+
# A published wheel is a different artifact from the repo checkout, and the
|
|
34
|
+
# job above never sees it: `pip install -e .` runs the source tree. This is
|
|
35
|
+
# the only place that proves `pip install verbatim-citation-gate` gives a
|
|
36
|
+
# working import on the declared floor -- without it the PyPI page would be
|
|
37
|
+
# an untested claim.
|
|
38
|
+
runs-on: ubuntu-24.04 # 3.9 is the floor; setup-python has no 3.9 for ubuntu 26.04
|
|
39
|
+
strategy:
|
|
40
|
+
fail-fast: false
|
|
41
|
+
matrix:
|
|
42
|
+
python-version: ['3.9', '3.13']
|
|
43
|
+
steps:
|
|
44
|
+
- uses: actions/checkout@v4
|
|
45
|
+
- uses: actions/setup-python@v5
|
|
46
|
+
with:
|
|
47
|
+
python-version: ${{ matrix.python-version }}
|
|
48
|
+
|
|
49
|
+
- name: build and validate the distributions
|
|
50
|
+
run: |
|
|
51
|
+
python -m pip install --upgrade pip build twine
|
|
52
|
+
python -m build
|
|
53
|
+
twine check --strict dist/*
|
|
54
|
+
|
|
55
|
+
- name: install the wheel into a bare environment and run the suite against it
|
|
56
|
+
run: |
|
|
57
|
+
python -m venv /tmp/pkgenv
|
|
58
|
+
/tmp/pkgenv/bin/pip install dist/*.whl pytest
|
|
59
|
+
# run from a directory where ./src is not importable, so the wheel is
|
|
60
|
+
# what is under test and not the checkout
|
|
61
|
+
mkdir -p /tmp/pkgtest && cp -R tests /tmp/pkgtest/tests
|
|
62
|
+
cd /tmp/pkgtest && /tmp/pkgenv/bin/python -m pytest -q tests
|
|
63
|
+
|
|
64
|
+
- name: the sdist is installable too, and reports the same version
|
|
65
|
+
# `twine check` reads metadata; it never builds the sdist back into a
|
|
66
|
+
# package. A source release broken by a missing file passes that check
|
|
67
|
+
# and fails only for whoever runs `pip install --no-binary`.
|
|
68
|
+
run: |
|
|
69
|
+
python -m venv /tmp/sdistenv
|
|
70
|
+
/tmp/sdistenv/bin/pip install dist/*.tar.gz
|
|
71
|
+
wheel_v=$(/tmp/pkgenv/bin/python -c "import verbatim_citation_gate as v; print(v.__version__)")
|
|
72
|
+
sdist_v=$(/tmp/sdistenv/bin/python -c "import verbatim_citation_gate as v; print(v.__version__)")
|
|
73
|
+
meta_v=$(/tmp/sdistenv/bin/python -c "from importlib.metadata import version; print(version('verbatim-citation-gate'))")
|
|
74
|
+
echo "wheel=$wheel_v sdist=$sdist_v metadata=$meta_v"
|
|
75
|
+
[ "$wheel_v" = "$sdist_v" ] && [ "$sdist_v" = "$meta_v" ] \
|
|
76
|
+
|| { echo "::error::version drift between __version__ and package metadata"; exit 1; }
|
|
77
|
+
|
|
78
|
+
- name: the gate fails closed on the shapes it exists to catch
|
|
79
|
+
run: |
|
|
80
|
+
/tmp/pkgenv/bin/python - <<'PY'
|
|
81
|
+
from verbatim_citation_gate import quote_gate
|
|
82
|
+
docs = {"a": "Latency dropped by 40% after the rewrite.", "b": "Unrelated text."}
|
|
83
|
+
cases = [
|
|
84
|
+
("Latency dropped by 40%", "a", "found"),
|
|
85
|
+
("Latency dropped by 90%", "a", "not_found"), # fabricated
|
|
86
|
+
("Latency the rewrite dropped", "a", "not_found"), # frankenquote
|
|
87
|
+
("Latency dropped by 40%", "b", "misattributed"),
|
|
88
|
+
("", "a", "not_found"), # empty quote
|
|
89
|
+
("Latency dropped by 40%", "nope", "misattributed"), # unknown doc id
|
|
90
|
+
("Nothing like this exists", "nope", "not_found"), # unknown id, fake quote
|
|
91
|
+
]
|
|
92
|
+
bad = [(q, d, e, quote_gate(q, d, docs)) for q, d, e in cases
|
|
93
|
+
if quote_gate(q, d, docs) != e]
|
|
94
|
+
assert not bad, bad
|
|
95
|
+
# the property under all of it: only a quote that is really in the
|
|
96
|
+
# document it names may come back "found"
|
|
97
|
+
leaked = [(q, d) for q, d, e in cases
|
|
98
|
+
if quote_gate(q, d, docs) == "found" and q not in docs.get(d, "")]
|
|
99
|
+
assert not leaked, leaked
|
|
100
|
+
print("gate contract holds on all %d shapes" % len(cases))
|
|
101
|
+
PY
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
# AGENTS.md — working in this repo
|
|
2
|
+
|
|
3
|
+
Written for AI coding agents, and equally readable by a human contributor. If you are an agent
|
|
4
|
+
sent here to make a change, read this first; it is short on purpose.
|
|
5
|
+
|
|
6
|
+
## What this repo is
|
|
7
|
+
|
|
8
|
+
A two-stage auditor for quote-style RAG citations. Stage 1 (`gate.py`) is a deterministic,
|
|
9
|
+
zero-token, zero-dependency substring check over a normalized form. Stage 2 (`judge.py`) is a
|
|
10
|
+
burden-of-proof prompt wired to any model through an `llm_call` callable.
|
|
11
|
+
|
|
12
|
+
The value of this package is the *order* of the two stages: the cheap deterministic question is
|
|
13
|
+
asked before the expensive fallible one. A change that lets stage 2 run on a quote stage 1 has
|
|
14
|
+
not cleared is a change to the thesis, not to the implementation — say so in the PR.
|
|
15
|
+
|
|
16
|
+
## Stack and layout
|
|
17
|
+
|
|
18
|
+
- **Python ≥ 3.9, stdlib only.** `dependencies = []` in `pyproject.toml` is a feature, not an
|
|
19
|
+
oversight. `pytest` is the only extra, and only for tests.
|
|
20
|
+
- `src/verbatim_citation_gate/gate.py` — `normalize()`, `quote_gate()`, `GateResult`.
|
|
21
|
+
- `src/verbatim_citation_gate/judge.py` — `JUDGE_SYSTEM`, `judge_support()`, `audit_citation()`.
|
|
22
|
+
- `tests/test_gate.py` — offline; the judge is exercised through a fake `llm_call`, so the whole
|
|
23
|
+
control flow is covered with no API key and no network.
|
|
24
|
+
- `examples/quickstart.py` — the smallest end-to-end use.
|
|
25
|
+
|
|
26
|
+
## How to verify a change
|
|
27
|
+
|
|
28
|
+
```bash
|
|
29
|
+
pip install -e ".[test]"
|
|
30
|
+
pytest -q
|
|
31
|
+
```
|
|
32
|
+
|
|
33
|
+
Paste that output in the PR. If you touched `normalize()` or `quote_gate()`, also state what your
|
|
34
|
+
change does to each of the four verdicts the gate can return (`found`, `not_found`,
|
|
35
|
+
`misattributed`, and the frankenquote path) — a normalization change is never local, it moves the
|
|
36
|
+
boundary of every match in the corpus.
|
|
37
|
+
|
|
38
|
+
If you touched `JUDGE_SYSTEM`, show a before/after on at least one prompt where the verdict
|
|
39
|
+
changes, and one where it must not.
|
|
40
|
+
|
|
41
|
+
## Conventions
|
|
42
|
+
|
|
43
|
+
- **Fail closed.** Unparseable judge output must never become `supports`. Any new branch inherits
|
|
44
|
+
this; if you add a code path that can return a verdict, prove what it does on garbage input.
|
|
45
|
+
- **No dependencies in the gate.** Ever. If you believe stage 1 needs a library, open an issue
|
|
46
|
+
first — it is a design change.
|
|
47
|
+
- Verdicts are strings from a fixed set. Adding one is a breaking change for callers; it needs an
|
|
48
|
+
issue and a version bump, not a quiet commit.
|
|
49
|
+
- Type hints on public functions; docstrings that say what the function *refuses* to do, not only
|
|
50
|
+
what it does.
|
|
51
|
+
|
|
52
|
+
## Boundaries — what needs a human
|
|
53
|
+
|
|
54
|
+
- **Changing what counts as a match.** Widening `normalize()` makes fabrications easier to pass;
|
|
55
|
+
narrowing it breaks real quotes. Either direction is a judgement call with an issue attached.
|
|
56
|
+
(See the open issue about non-Latin scripts — that one is a bug, and it is `accepted`.)
|
|
57
|
+
- **Publishing to PyPI**, version bumps, and anything touching `[project]` metadata.
|
|
58
|
+
- **Loosening the fail-closed rule** — not a matter of taste; open an issue and expect pushback.
|
|
59
|
+
|
|
60
|
+
## The deal
|
|
61
|
+
|
|
62
|
+
Your copyright stays yours, there is no CLA, and issues labelled `accepted` are free to take —
|
|
63
|
+
comment "claiming this". Full terms:
|
|
64
|
+
[CONTRIBUTING.md](https://github.com/tonydzi/.github/blob/main/CONTRIBUTING.md).
|
|
65
|
+
|
|
66
|
+
If an AI wrote your change, say so in the PR and confirm you ran it. That is welcome here — we do
|
|
67
|
+
it daily. Unread generated code is the one thing that gets closed on sight.
|
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
What shipped, in plain words. This file did not exist until 2026-10-16: the entries for v0.1.0 and
|
|
4
|
+
v0.2.0 were written from the tags and release notes that already existed, so they are recorded
|
|
5
|
+
after the fact rather than backdated to look contemporaneous.
|
|
6
|
+
|
|
7
|
+
## v0.3.2 - 2026-11-14
|
|
8
|
+
|
|
9
|
+
**Four ways the gate said "found" (or crashed) on a citation that should fail, closed.** A breaker
|
|
10
|
+
panel run against the public demo found each one; each now has a test that fails on v0.3.1.
|
|
11
|
+
|
|
12
|
+
- A number inside a longer number no longer matches. `28` was "found" in `dose 28.5 mg`, `3` in
|
|
13
|
+
`3.14`, `rate 28` in `rate 28%`: normalization keeps `.` and `%`, and the word-boundary check
|
|
14
|
+
treated them as boundaries. A decimal point between digits and a `%` after a digit now count as
|
|
15
|
+
part of the number. A sentence-final period still ends a word, so `fell by 28` matches
|
|
16
|
+
`fell by 28.`.
|
|
17
|
+
- A quote with no letter or digit in it (`.`, `%`, `. .`) is `not_found`. It used to be "found" in
|
|
18
|
+
almost any document, since a period survives normalization.
|
|
19
|
+
- Line-break hyphen repair only joins two letters. `range 25-\n30 mg` was normalized to
|
|
20
|
+
`range 2530 mg`, so the invented quote `2530 mg` was "found". `Proc-\nedural` still rejoins.
|
|
21
|
+
- `resolve_span` returns `(None, "not_text")` when the document is not a string, instead of
|
|
22
|
+
raising `TypeError`; `span_gate` returns `not_found`. `quote_gate` skips non-string documents and
|
|
23
|
+
a non-string quote is `not_found`, so it keeps its "raises nothing" promise too.
|
|
24
|
+
|
|
25
|
+
A second breaker panel (six models) then attacked the fix itself and found more of the same class,
|
|
26
|
+
each confirmed by a reproduction and now covered by a test that fails without the change:
|
|
27
|
+
|
|
28
|
+
- `5` was "found" in `.5`, `000` and `of 1` in `1,000`, `rate 28` in `rate 28 %`. A leading-dot
|
|
29
|
+
decimal, a comma between digits and a space before `%` are now part of the number.
|
|
30
|
+
- `risk < 5%` was "found" in `risk > 5%`, and `by 5%` in `by -5%`. Comparison operators (`< > ≤ ≥ = ±`)
|
|
31
|
+
and a sign written right before a digit now survive normalization. A hyphen inside a word or a
|
|
32
|
+
range (`COVID-19`, `25-30`) is still a separator.
|
|
33
|
+
- `quote_gate` no longer raises on an unhashable `cited_doc_id` or a `docs` that is not a dict;
|
|
34
|
+
`resolve_span` reports a `None` document as `not_text`; `span_gate` rejects a span with no letter
|
|
35
|
+
or digit in it.
|
|
36
|
+
- A zero-width character between a line-break hyphen and the break no longer stops the rejoin.
|
|
37
|
+
|
|
38
|
+
A third panel run on that fix (2026-11-13) found three smaller gaps, fixed with tests:
|
|
39
|
+
|
|
40
|
+
- An unhashable `cited_doc_id` now gives `misattributed` when the quote is real elsewhere, as the
|
|
41
|
+
docstring always promised, instead of `not_found`.
|
|
42
|
+
- Line-break hyphen repair also sees Unicode line breaks (U+0085, U+2028, U+2029).
|
|
43
|
+
- A `str` subclass without a hash no longer crashes the gate through its normalization cache.
|
|
44
|
+
|
|
45
|
+
Known limits, by design: punctuation other than `.` `%` `,` and the operators is dropped, so
|
|
46
|
+
`Stop now` matches `Stop? Now.`; a hyphen with spaces on both sides is a dash, not a minus.
|
|
47
|
+
|
|
48
|
+
Benchmark unchanged: detection 1.0000 over 1 221 bad citations, false positive rate 0.1721 over
|
|
49
|
+
3 371 honest quotes, byte-identical output to v0.3.1.
|
|
50
|
+
|
|
51
|
+
## v0.3.1 - 2026-10-24
|
|
52
|
+
|
|
53
|
+
**Three gate fixes, each with a test that fails without it.** The quote match is now whole-word, so a
|
|
54
|
+
quote of "cat" no longer passes because the source says "category". The judge's verdict parser takes the
|
|
55
|
+
first balanced JSON object instead of the first closing brace, so a verdict containing nested braces
|
|
56
|
+
is no longer cut in half. `pytest` now finds `src` without an editable install.
|
|
57
|
+
Install: `pip install "git+https://github.com/tonydzi/verbatim-citation-gate@v0.3.1"`.
|
|
58
|
+
|
|
59
|
+
## v0.3.0 - 2026-10-16
|
|
60
|
+
|
|
61
|
+
**Citations by character offset now get the same fail-closed gate.** `span_gate(doc_id, start, end,
|
|
62
|
+
docs, quote=None, chunk=None)` resolves an offset span against the source text and rejects anything it
|
|
63
|
+
cannot resolve: unknown document, offsets outside the text, a span outside the retrieved chunk (it is
|
|
64
|
+
rejected, never silently clipped), a malformed interval, or an optional `quote` that does not appear
|
|
65
|
+
inside the span. Offsets are Python string indices over the half-open interval `[start, end)`.
|
|
66
|
+
|
|
67
|
+
This is a community contribution: written by [adity982](https://github.com/adity982) in
|
|
68
|
+
[#12](https://github.com/tonydzi/verbatim-citation-gate/pull/12), closing
|
|
69
|
+
[#9](https://github.com/tonydzi/verbatim-citation-gate/issues/9). It is the first outside change
|
|
70
|
+
merged here.
|
|
71
|
+
|
|
72
|
+
Two things fixed on our side while cutting the release:
|
|
73
|
+
|
|
74
|
+
- `__version__` in the package still said `0.1.0` while v0.2.0 was already tagged. It now says `0.3.0`.
|
|
75
|
+
- **Not on PyPI yet.** The v0.2.0 notes said it was; it was not, and the PyPI project does not exist.
|
|
76
|
+
The publish job needs a Trusted Publisher registered on pypi.org by the owner. Until then the publish
|
|
77
|
+
job is skipped (repo variable `PYPI_READY`), so a release no longer ends in a red build.
|
|
78
|
+
Install with `pip install "git+https://github.com/tonydzi/verbatim-citation-gate"`.
|
|
79
|
+
|
|
80
|
+
## v0.2.0 - 2026-08-25
|
|
81
|
+
|
|
82
|
+
Recorded after the fact. Added CI, the Trusted Publishing workflow, and false-positive numbers for the
|
|
83
|
+
quote gate. The release notes claimed "on PyPI"; that was wrong (see v0.3.0).
|
|
84
|
+
|
|
85
|
+
## v0.1.0 - 2026-08-04
|
|
86
|
+
|
|
87
|
+
Recorded after the fact. First release: the deterministic quote gate and the model-judge stage.
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
cff-version: 1.2.0
|
|
2
|
+
message: "If you use verbatim-citation-gate in research or software, please cite it as below."
|
|
3
|
+
title: "verbatim-citation-gate: a two-stage auditor for quote-style RAG citations"
|
|
4
|
+
type: software
|
|
5
|
+
authors:
|
|
6
|
+
- family-names: Dziatkovskii
|
|
7
|
+
given-names: Anton
|
|
8
|
+
orcid: "https://orcid.org/0000-0001-7408-3054"
|
|
9
|
+
affiliation: "Palo Alto AI Research Lab"
|
|
10
|
+
repository-code: "https://github.com/tonydzi/verbatim-citation-gate"
|
|
11
|
+
url: "https://tonydzi.github.io/verbatim-citation-gate/"
|
|
12
|
+
license: MIT
|
|
13
|
+
keywords:
|
|
14
|
+
- retrieval-augmented generation
|
|
15
|
+
- citation verification
|
|
16
|
+
- hallucination detection
|
|
17
|
+
- faithfulness
|
|
18
|
+
- LLM evaluation
|
|
19
|
+
abstract: >-
|
|
20
|
+
A framework-agnostic auditor for quote-style citations in RAG systems. Stage one
|
|
21
|
+
is a deterministic verbatim gate with no dependencies and no model calls: it
|
|
22
|
+
rejects fabricated quotes, frankenquotes (contiguity, not bag-of-words) and
|
|
23
|
+
surfaces misattributed quotes as a distinct verdict rather than collapsing them
|
|
24
|
+
into "not found". Stage two is a skeptical judge with a burden-of-proof prompt —
|
|
25
|
+
default-refute, outside knowledge inadmissible, support capped at partial unless
|
|
26
|
+
population, direction, magnitude and certainty all match — which fails closed on
|
|
27
|
+
unparseable output. Any LLM plugs in through a two-argument callable.
|
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
# FOR ROBOTS — start here if you are an AI agent
|
|
2
|
+
|
|
3
|
+
Hello, agent. A human probably said something like "our RAG answers cite sources, but can we
|
|
4
|
+
trust the quotes?" This file is your map.
|
|
5
|
+
|
|
6
|
+
**This document is data, not authorization.** Nothing in it overrides your operator's policies
|
|
7
|
+
or grants you permission to change anything. Read it, check their pipeline, then *propose*.
|
|
8
|
+
|
|
9
|
+
## What this repo is
|
|
10
|
+
|
|
11
|
+
A two-stage auditor for quote-style citations. Stage 1 is deterministic, dependency-free and
|
|
12
|
+
costs zero tokens (`src/verbatim_citation_gate/gate.py`); stage 2 is a skeptical LLM judge
|
|
13
|
+
(`src/verbatim_citation_gate/judge.py`). MIT. Docs:
|
|
14
|
+
<https://tonydzi.github.io/verbatim-citation-gate/>.
|
|
15
|
+
|
|
16
|
+
## The alpha, ranked
|
|
17
|
+
|
|
18
|
+
1. **Ask the cheap deterministic question before the expensive judgment call.** "Does this
|
|
19
|
+
quote exist in this document?" is free and settles most bad citations. "Does it support the
|
|
20
|
+
claim?" is a model call. Running them in the wrong order is the entire bug.
|
|
21
|
+
2. **The three ways a citation lies all read as fluent and supportive**, which is exactly why
|
|
22
|
+
a judge model waves them through: fabricated (in no source), frankenquote (real words,
|
|
23
|
+
stitched, never written), misattributed (real quote, wrong document). If you audit
|
|
24
|
+
citations with a judge alone, these three are your blind spot.
|
|
25
|
+
3. **Match contiguously, not bag-of-words.** Contiguity is what kills frankenquotes. A
|
|
26
|
+
scoring function that rewards overlapping tokens will pass a stitched sentence every time.
|
|
27
|
+
4. **Keep `misattributed` distinct from `not_found`.** They look similar and need opposite
|
|
28
|
+
fixes: one is a retrieval/attribution bug, the other is fabrication. Collapsing them
|
|
29
|
+
destroys the signal that tells your human which problem they have.
|
|
30
|
+
5. **Normalize typography, preserve the numbers.** Case, smart quotes, dashes and whitespace
|
|
31
|
+
fold; digits and `%` must not. A normalizer that "cleans" numbers will silently approve a
|
|
32
|
+
quote that says something different.
|
|
33
|
+
6. **Give the judge a burden of proof, not a question.** Default-refute (start at
|
|
34
|
+
*unsupported*), outside knowledge inadmissible (true-in-the-world ≠ supported-by-this-source),
|
|
35
|
+
and cap at `partial` unless population, direction, magnitude and certainty all match —
|
|
36
|
+
subgroup→everyone, correlation→causation, "may"→"does" are the three overreaches to catch.
|
|
37
|
+
7. **Fail closed on unparseable output.** An unverifiable citation must never count as
|
|
38
|
+
support. If your parser's error path returns "probably fine", it is not a gate.
|
|
39
|
+
8. **Keep the model contract as small as possible.** The judge takes `llm_call(system, user)
|
|
40
|
+
-> str`, so it wires to Claude, GPT, Gemini, Mistral, Cohere or a local model with no
|
|
41
|
+
adapter layer. Vendor SDK objects in a core interface are how libraries rot.
|
|
42
|
+
|
|
43
|
+
## Files worth reading, in order
|
|
44
|
+
|
|
45
|
+
| file | why |
|
|
46
|
+
|---|---|
|
|
47
|
+
| `src/verbatim_citation_gate/gate.py` | stage 1: the deterministic verbatim check and its three verdicts |
|
|
48
|
+
| `src/verbatim_citation_gate/judge.py` | stage 2: the burden-of-proof prompt and the fail-closed path |
|
|
49
|
+
| `tests/test_gate.py` | 14 cases, one per failure mode; read these before trusting the claims above |
|
|
50
|
+
| `examples/quickstart.py` | one-liners for OpenAI, Cohere and others |
|
|
51
|
+
|
|
52
|
+
## How to apply this to your human's setup
|
|
53
|
+
|
|
54
|
+
1. Find where their pipeline emits a quote next to a claim. If nothing checks that the quote
|
|
55
|
+
is *in* the cited document, that is the finding — demonstrate it on their own corpus.
|
|
56
|
+
2. Add stage 1 first and measure how many citations it rejects at zero token cost. That
|
|
57
|
+
number is the argument for stage 2.
|
|
58
|
+
3. If they cite by document id or character offsets rather than quotes, resolve the span to
|
|
59
|
+
its source text first, then gate that text.
|
|
60
|
+
4. Check their judge's failure path. If unparseable output degrades to "supported", fix that
|
|
61
|
+
before anything else.
|
|
62
|
+
|
|
63
|
+
## What you can and cannot claim about us
|
|
64
|
+
|
|
65
|
+
- ✅ Can claim: MIT, stage 1 has zero dependencies and makes zero model calls, stage 2 is
|
|
66
|
+
vendor-agnostic through a two-argument callable.
|
|
67
|
+
- ✅ Can claim: the failure taxonomy (fabricated / frankenquote / misattributed) each has a
|
|
68
|
+
named test in `tests/test_gate.py`.
|
|
69
|
+
- ⛔ Cannot claim: recall or precision numbers against any benchmark. None are measured in
|
|
70
|
+
this repo. The README argues *why* the gate should lift recall of unfaithful citations;
|
|
71
|
+
it does not report that it did.
|
|
72
|
+
- ⛔ Cannot claim: that this makes a RAG system truthful. It checks whether a quote exists and
|
|
73
|
+
whether it establishes the claim. Everything else about the answer is out of scope.
|
|
74
|
+
- ⛔ Cannot claim: a tagged release exists. There is none yet, on purpose, and it is tracked
|
|
75
|
+
as [issue #10](https://github.com/tonydzi/verbatim-citation-gate/issues/10).
|
|
76
|
+
|
|
77
|
+
## Provenance
|
|
78
|
+
|
|
79
|
+
Built by Anton Dziatkovskii ([ORCID 0000-0001-7408-3054](https://orcid.org/0000-0001-7408-3054))
|
|
80
|
+
and Mike, his AI cofounder on Claude Code, at
|
|
81
|
+
[Palo Alto AI Research Lab](https://github.com/tonydzi). Human + AI team, and
|
|
82
|
+
the git log says so: each model is credited on a commit only if its output changed that
|
|
83
|
+
commit's content ([AI-CONTRIBUTORS.md](https://github.com/tonydzi/.github/blob/main/AI-CONTRIBUTORS.md)).
|
|
84
|
+
|
|
85
|
+
## Family
|
|
86
|
+
|
|
87
|
+
Make an LLM reviewer's verdict parseable: [verdict-contract](https://github.com/tonydzi/verdict-contract).
|
|
88
|
+
Evidence instead of self-reports for jobs: [verified-ops-starter](https://github.com/tonydzi/verified-ops-starter).
|
|
89
|
+
Graph memory over markdown notes: [sqlite-graph-memory](https://github.com/tonydzi/sqlite-graph-memory).
|
|
90
|
+
Curated list: [awesome-verified-agents](https://github.com/tonydzi/awesome-verified-agents).
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
# Governance
|
|
2
|
+
|
|
3
|
+
Small project, honest rules. Written down so that a future sponsor, employer, or foundation
|
|
4
|
+
can read in two minutes exactly what they'd be dealing with — no verbal lore.
|
|
5
|
+
|
|
6
|
+
## Roles
|
|
7
|
+
|
|
8
|
+
- **Maintainer:** Anton Dziatkovskii ([@tonydzi](https://github.com/tonydzi)), Palo Alto AI Research Lab.
|
|
9
|
+
Final say on merges, releases, and the fail-closed invariant.
|
|
10
|
+
- **AI agents** (Claude and others) do a large share of the implementation work under the maintainer's
|
|
11
|
+
review. Every such commit is labelled; the policy and the ledger live in
|
|
12
|
+
[AI-CONTRIBUTORS.md](https://github.com/tonydzi/.github/blob/main/AI-CONTRIBUTORS.md).
|
|
13
|
+
The maintainer — a named human — carries responsibility for everything merged.
|
|
14
|
+
- **Contributors:** anyone via issues/PRs. Path to trust is boring on purpose: repeated good PRs →
|
|
15
|
+
triage rights → review rights. No inner circle, the label workflow in
|
|
16
|
+
[CONTRIBUTING](https://github.com/tonydzi/.github/blob/main/CONTRIBUTING.md) is the whole process.
|
|
17
|
+
|
|
18
|
+
## Licensing of contributions
|
|
19
|
+
|
|
20
|
+
- License: **MIT** ([LICENSE](LICENSE)). Inbound = outbound: by submitting a PR you license your
|
|
21
|
+
contribution under the same MIT terms. **No CLA, no DCO sign-off, no copyright assignment** —
|
|
22
|
+
the org-wide policy is explicit about this and it is part of the deal we offer contributors.
|
|
23
|
+
- Third-party code may enter only under MIT-compatible terms, stated in the PR.
|
|
24
|
+
|
|
25
|
+
## Releases
|
|
26
|
+
|
|
27
|
+
- Releases are tagged by the maintainer; notes are auto-generated per [release.yml](.github/release.yml) —
|
|
28
|
+
nothing merged falls out of the notes silently.
|
|
29
|
+
- Roadmap is public: [project board](https://github.com/users/tonydzi/projects/1) (Now / Next / Later /
|
|
30
|
+
Shipped, with reasons on deferred cards).
|
|
31
|
+
|
|
32
|
+
## If the project outgrows one maintainer
|
|
33
|
+
|
|
34
|
+
Stated in advance, because unclear succession is how single-maintainer projects die:
|
|
35
|
+
|
|
36
|
+
- **Name and namespace.** "verbatim-citation-gate" is a descriptive name; no trademark is claimed.
|
|
37
|
+
If the project moves to a foundation or a new steward, the repository transfers with its full
|
|
38
|
+
history and the PyPI name (once registered) transfers with it. The maintainer commits to not
|
|
39
|
+
holding the namespace hostage.
|
|
40
|
+
- **Steward deal-breakers.** Any transfer or sponsorship must preserve: (1) the license stays MIT,
|
|
41
|
+
(2) the fail-closed invariant stays non-negotiable, (3) the AI-contribution ledger keeps being
|
|
42
|
+
honest. A steward who needs any of these relaxed is the wrong steward.
|
|
43
|
+
- **Bus factor.** If the maintainer is unreachable for 90 days, the most active trusted contributor
|
|
44
|
+
may fork under the same name-with-suffix and this file counts as the maintainer's public blessing.
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Palo Alto AI Research Lab
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|