sentinel-proof 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sentinel_proof/__init__.py +51 -0
- sentinel_proof/citations.py +67 -0
- sentinel_proof/cli.py +117 -0
- sentinel_proof/grounding.py +483 -0
- sentinel_proof/passages.py +148 -0
- sentinel_proof/provenance.py +32 -0
- sentinel_proof/py.typed +0 -0
- sentinel_proof/refusal.py +58 -0
- sentinel_proof-0.1.0.dist-info/METADATA +201 -0
- sentinel_proof-0.1.0.dist-info/RECORD +14 -0
- sentinel_proof-0.1.0.dist-info/WHEEL +5 -0
- sentinel_proof-0.1.0.dist-info/entry_points.txt +2 -0
- sentinel_proof-0.1.0.dist-info/licenses/LICENSE +202 -0
- sentinel_proof-0.1.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
"""sentinel-proof: a deterministic grounding gate for RAG answers.
|
|
2
|
+
|
|
3
|
+
from sentinel_proof import verify, passages_from_texts
|
|
4
|
+
|
|
5
|
+
passages = passages_from_texts(["Customers may request a refund within 30 days."])
|
|
6
|
+
v = verify("Refunds are available for 60 days [1].", passages, citation_style="numeric")
|
|
7
|
+
v.ok # False: the only claim's number is not in the evidence
|
|
8
|
+
v.safe_text() # refusal prose generated from the reason code, never the draft
|
|
9
|
+
|
|
10
|
+
Pure Python, no dependencies, no model calls. See the README for the method.
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
from .citations import CitationStyle, normalize_citations, strip_citations
|
|
14
|
+
from .grounding import (
|
|
15
|
+
MAX_STRIPPED_RATIO,
|
|
16
|
+
MIN_LEXICAL_OVERLAP,
|
|
17
|
+
Claim,
|
|
18
|
+
ClaimStatus,
|
|
19
|
+
Verdict,
|
|
20
|
+
verify,
|
|
21
|
+
)
|
|
22
|
+
from .passages import Passage, from_langchain, from_llamaindex, passages_from_texts
|
|
23
|
+
from .provenance import HASH_RECIPE, check_provenance, document_sha256, provenance_hash
|
|
24
|
+
from .refusal import REFUSAL_SENTINEL, RefusalReason, is_refusal, refusal_text
|
|
25
|
+
|
|
26
|
+
__version__ = "0.1.0"
|
|
27
|
+
|
|
28
|
+
__all__ = [
|
|
29
|
+
"HASH_RECIPE",
|
|
30
|
+
"MAX_STRIPPED_RATIO",
|
|
31
|
+
"MIN_LEXICAL_OVERLAP",
|
|
32
|
+
"REFUSAL_SENTINEL",
|
|
33
|
+
"CitationStyle",
|
|
34
|
+
"Claim",
|
|
35
|
+
"ClaimStatus",
|
|
36
|
+
"Passage",
|
|
37
|
+
"RefusalReason",
|
|
38
|
+
"Verdict",
|
|
39
|
+
"__version__",
|
|
40
|
+
"check_provenance",
|
|
41
|
+
"document_sha256",
|
|
42
|
+
"from_langchain",
|
|
43
|
+
"from_llamaindex",
|
|
44
|
+
"is_refusal",
|
|
45
|
+
"normalize_citations",
|
|
46
|
+
"passages_from_texts",
|
|
47
|
+
"provenance_hash",
|
|
48
|
+
"refusal_text",
|
|
49
|
+
"strip_citations",
|
|
50
|
+
"verify",
|
|
51
|
+
]
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
"""Citation tags: the shapes models write them in, and one canonical form.
|
|
2
|
+
|
|
3
|
+
Internally every citation is ``[chunk:<passage id>]``. Two things feed that:
|
|
4
|
+
|
|
5
|
+
* **Tag style** (``[chunk:id]``), which models also write as ``【chunk:id】``
|
|
6
|
+
(gpt-oss, trained on OpenAI's own citation glyphs), ``[ chunk: id ]`` or
|
|
7
|
+
with full-width brackets. All are normalised; none of them should turn a
|
|
8
|
+
correct, fully cited answer into a refusal.
|
|
9
|
+
* **Numeric style** (``[1]``, ``[2, 3]``, ``[1][2]``), the convention most RAG
|
|
10
|
+
prompts use, where the number is the passage's 1-based position in the
|
|
11
|
+
context. Converted to tags for checking and back to numbers for output.
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
from __future__ import annotations
|
|
15
|
+
|
|
16
|
+
import re
|
|
17
|
+
from collections.abc import Sequence
|
|
18
|
+
from typing import Literal
|
|
19
|
+
|
|
20
|
+
CitationStyle = Literal["tag", "numeric"]
|
|
21
|
+
|
|
22
|
+
CITATION_RE = re.compile(r"\[chunk:([\w-]+)\]")
|
|
23
|
+
_ALT_CITATION_RE = re.compile(r"[\[【[〔]\s*chunk\s*:\s*([\w-]+)\s*[\]】]〕]")
|
|
24
|
+
# "[1]", "[1, 2]", "[1][2]", "【3】": digits and separators only, so a bracketed
|
|
25
|
+
# word in ordinary prose ("[sic]") is never read as a citation.
|
|
26
|
+
_NUMERIC_GROUP_RE = re.compile(r"[\[【[]\s*(\d+(?:\s*[,;]\s*\d+)*)\s*[\]】]]")
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def normalize_citations(text: str) -> str:
|
|
30
|
+
"""Rewrite every recognised tag-style citation to the canonical ``[chunk:id]``."""
|
|
31
|
+
return _ALT_CITATION_RE.sub(r"[chunk:\1]", text)
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def numeric_to_tags(text: str, ids: Sequence[str]) -> str:
|
|
35
|
+
"""``[1, 3]`` -> ``[chunk:<ids[0]>] [chunk:<ids[2]>]``.
|
|
36
|
+
|
|
37
|
+
A number with no passage behind it (``[9]`` when five were retrieved) is
|
|
38
|
+
kept as a tag naming a passage that does not exist, so the gate treats it
|
|
39
|
+
exactly like any other fabricated citation rather than silently dropping it.
|
|
40
|
+
"""
|
|
41
|
+
|
|
42
|
+
def repl(m: re.Match[str]) -> str:
|
|
43
|
+
tags: list[str] = []
|
|
44
|
+
for n in re.split(r"\s*[,;]\s*", m.group(1)):
|
|
45
|
+
i = int(n)
|
|
46
|
+
tags.append(f"[chunk:{ids[i - 1]}]" if 1 <= i <= len(ids) else f"[chunk:missing-{i}]")
|
|
47
|
+
return " ".join(tags)
|
|
48
|
+
|
|
49
|
+
return _NUMERIC_GROUP_RE.sub(repl, text)
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def tags_to_numeric(text: str, ids: Sequence[str]) -> str:
|
|
53
|
+
"""``[chunk:<ids[2]>]`` -> ``[3]``. Adjacent tags become ``[1][3]``."""
|
|
54
|
+
position = {pid: i + 1 for i, pid in enumerate(ids)}
|
|
55
|
+
|
|
56
|
+
def repl(m: re.Match[str]) -> str:
|
|
57
|
+
return f"[{position[m.group(1)]}]" if m.group(1) in position else ""
|
|
58
|
+
|
|
59
|
+
out = CITATION_RE.sub(repl, text)
|
|
60
|
+
return re.sub(r"\]\s+\[", "][", out)
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def strip_citations(text: str) -> str:
|
|
64
|
+
"""The prose alone, for display surfaces that render citations separately."""
|
|
65
|
+
out = CITATION_RE.sub("", normalize_citations(text))
|
|
66
|
+
out = re.sub(r"\s+([,.;:!?])", r"\1", out)
|
|
67
|
+
return re.sub(r"[ \t]{2,}", " ", out).strip()
|
sentinel_proof/cli.py
ADDED
|
@@ -0,0 +1,117 @@
|
|
|
1
|
+
"""``sentinel-proof``: run the gate from a shell or a CI step.
|
|
2
|
+
|
|
3
|
+
sentinel-proof check --answer answer.txt --passages passages.json
|
|
4
|
+
sentinel-proof check --answer answer.txt --passages passages.json --numeric --json
|
|
5
|
+
echo "Refunds take 30 days [1]." | sentinel-proof check --passages passages.json --numeric
|
|
6
|
+
|
|
7
|
+
``passages.json`` is a list of strings, or a list of objects with ``text`` and
|
|
8
|
+
any other :class:`Passage` field. Exit code 0 when the answer is verified,
|
|
9
|
+
1 when it is not, 2 for a usage error.
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
import argparse
|
|
15
|
+
import dataclasses
|
|
16
|
+
import json
|
|
17
|
+
import sys
|
|
18
|
+
from pathlib import Path
|
|
19
|
+
from typing import Any
|
|
20
|
+
|
|
21
|
+
from . import __version__
|
|
22
|
+
from .grounding import MAX_STRIPPED_RATIO, Verdict, verify
|
|
23
|
+
from .passages import Passage, _with_ids, passages_from_texts
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def _load_passages(path: Path) -> list[Passage]:
|
|
27
|
+
raw = json.loads(path.read_text(encoding="utf-8"))
|
|
28
|
+
if not isinstance(raw, list) or not raw:
|
|
29
|
+
raise SystemExit("passages file must be a non-empty JSON list")
|
|
30
|
+
if all(isinstance(x, str) for x in raw):
|
|
31
|
+
return passages_from_texts(raw)
|
|
32
|
+
fields = {f.name for f in dataclasses.fields(Passage)} - {"metadata"}
|
|
33
|
+
out: list[Passage] = []
|
|
34
|
+
for x in raw:
|
|
35
|
+
if isinstance(x, str):
|
|
36
|
+
x = {"text": x}
|
|
37
|
+
if not isinstance(x, dict) or "text" not in x:
|
|
38
|
+
raise SystemExit("each passage must be a string or an object with a 'text' field")
|
|
39
|
+
known = {k: v for k, v in x.items() if k in fields}
|
|
40
|
+
extra = {k: v for k, v in x.items() if k not in fields and k != "metadata"}
|
|
41
|
+
out.append(Passage(**known, metadata={**x.get("metadata", {}), **extra}))
|
|
42
|
+
return _with_ids(out)
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def _as_dict(v: Verdict) -> dict[str, Any]:
|
|
46
|
+
d = dataclasses.asdict(v)
|
|
47
|
+
d["refusal"] = v.refusal.value if v.refusal else None
|
|
48
|
+
d["passed"] = v.passed
|
|
49
|
+
d["safe_text"] = v.safe_text()
|
|
50
|
+
for c in d["claims"]:
|
|
51
|
+
c["status"] = str(c["status"])
|
|
52
|
+
return d
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def _print_human(v: Verdict) -> None:
|
|
56
|
+
if v.is_refusal:
|
|
57
|
+
print("REFUSED by the model: the evidence does not answer the question.")
|
|
58
|
+
return
|
|
59
|
+
label = "VERIFIED" if v.ok else "NOT VERIFIED"
|
|
60
|
+
print(
|
|
61
|
+
f"{label}: {len(v.claims) - len(v.stripped)}/{len(v.claims)} claims supported, "
|
|
62
|
+
f"{v.repaired_citations} citation(s) repaired, "
|
|
63
|
+
f"{v.stripped_ratio:.0%} stripped (limit {MAX_STRIPPED_RATIO:.0%})"
|
|
64
|
+
)
|
|
65
|
+
for c in v.claims:
|
|
66
|
+
mark = {"supported": "+", "repaired": "~", "stripped": "-"}[str(c.status)]
|
|
67
|
+
detail = ""
|
|
68
|
+
if c.status.value == "stripped" and c.unsupported_numbers:
|
|
69
|
+
detail = f" (figures not in cited evidence: {', '.join(c.unsupported_numbers)})"
|
|
70
|
+
elif c.status.value == "repaired":
|
|
71
|
+
cited = ", ".join(c.cited) or "nothing"
|
|
72
|
+
detail = f" (cited {cited}; supported by {', '.join(c.supported_by)})"
|
|
73
|
+
print(f" {mark} {c.text or '<no text>'}{detail}")
|
|
74
|
+
if v.ok:
|
|
75
|
+
print("\n" + v.answer)
|
|
76
|
+
else:
|
|
77
|
+
print("\n" + v.safe_text())
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def main(argv: list[str] | None = None) -> int:
|
|
81
|
+
parser = argparse.ArgumentParser(prog="sentinel-proof", description=__doc__.split("\n\n")[0])
|
|
82
|
+
parser.add_argument("--version", action="version", version=__version__)
|
|
83
|
+
sub = parser.add_subparsers(dest="command", required=True)
|
|
84
|
+
|
|
85
|
+
check = sub.add_parser("check", help="verify an answer against its passages")
|
|
86
|
+
check.add_argument("--answer", type=Path, help="file with the answer (default: stdin)")
|
|
87
|
+
check.add_argument("--passages", type=Path, required=True, help="JSON list of passages")
|
|
88
|
+
check.add_argument("--question", default="", help="the user's question, if any")
|
|
89
|
+
check.add_argument(
|
|
90
|
+
"--numeric", action="store_true", help="citations are [1], [2]... not [chunk:id]"
|
|
91
|
+
)
|
|
92
|
+
check.add_argument(
|
|
93
|
+
"--max-stripped",
|
|
94
|
+
type=float,
|
|
95
|
+
default=MAX_STRIPPED_RATIO,
|
|
96
|
+
help="fail when more than this fraction of claims is stripped",
|
|
97
|
+
)
|
|
98
|
+
check.add_argument("--json", action="store_true", help="machine-readable output")
|
|
99
|
+
|
|
100
|
+
args = parser.parse_args(argv)
|
|
101
|
+
answer = args.answer.read_text(encoding="utf-8") if args.answer else sys.stdin.read()
|
|
102
|
+
verdict = verify(
|
|
103
|
+
answer,
|
|
104
|
+
_load_passages(args.passages),
|
|
105
|
+
args.question,
|
|
106
|
+
citation_style="numeric" if args.numeric else "tag",
|
|
107
|
+
max_stripped_ratio=args.max_stripped,
|
|
108
|
+
)
|
|
109
|
+
if args.json:
|
|
110
|
+
print(json.dumps(_as_dict(verdict), indent=2))
|
|
111
|
+
else:
|
|
112
|
+
_print_human(verdict)
|
|
113
|
+
return 0 if verdict.passed else 1
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
if __name__ == "__main__": # pragma: no cover
|
|
117
|
+
sys.exit(main())
|