sourced-evidence 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sourced_evidence-0.1.0/LICENSE +21 -0
- sourced_evidence-0.1.0/PKG-INFO +153 -0
- sourced_evidence-0.1.0/README.md +132 -0
- sourced_evidence-0.1.0/pyproject.toml +40 -0
- sourced_evidence-0.1.0/setup.cfg +4 -0
- sourced_evidence-0.1.0/sourced/__init__.py +1 -0
- sourced_evidence-0.1.0/sourced/check.py +96 -0
- sourced_evidence-0.1.0/sourced/claims.py +44 -0
- sourced_evidence-0.1.0/sourced/cli.py +64 -0
- sourced_evidence-0.1.0/sourced/signals.py +67 -0
- sourced_evidence-0.1.0/sourced_evidence.egg-info/PKG-INFO +153 -0
- sourced_evidence-0.1.0/sourced_evidence.egg-info/SOURCES.txt +17 -0
- sourced_evidence-0.1.0/sourced_evidence.egg-info/dependency_links.txt +1 -0
- sourced_evidence-0.1.0/sourced_evidence.egg-info/entry_points.txt +2 -0
- sourced_evidence-0.1.0/sourced_evidence.egg-info/top_level.txt +1 -0
- sourced_evidence-0.1.0/tests/test_check.py +84 -0
- sourced_evidence-0.1.0/tests/test_claims.py +78 -0
- sourced_evidence-0.1.0/tests/test_cli.py +90 -0
- sourced_evidence-0.1.0/tests/test_signals.py +84 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Ritish Saini
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,153 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: sourced-evidence
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Per-claim grounded/contradicted/unverified checking of LLM output against its own source context -- never a blended trust score.
|
|
5
|
+
License-Expression: MIT
|
|
6
|
+
Project-URL: Homepage, https://github.com/MaXiMo000/sourced
|
|
7
|
+
Project-URL: Source, https://github.com/MaXiMo000/sourced
|
|
8
|
+
Project-URL: Issues, https://github.com/MaXiMo000/sourced/issues
|
|
9
|
+
Project-URL: Changelog, https://github.com/MaXiMo000/sourced/releases
|
|
10
|
+
Keywords: verification,grounding,llm,rag,evidence,provenance
|
|
11
|
+
Classifier: Development Status :: 3 - Alpha
|
|
12
|
+
Classifier: Environment :: Console
|
|
13
|
+
Classifier: Intended Audience :: Developers
|
|
14
|
+
Classifier: Operating System :: OS Independent
|
|
15
|
+
Classifier: Programming Language :: Python :: 3
|
|
16
|
+
Classifier: Topic :: Software Development :: Testing
|
|
17
|
+
Requires-Python: >=3.10
|
|
18
|
+
Description-Content-Type: text/markdown
|
|
19
|
+
License-File: LICENSE
|
|
20
|
+
Dynamic: license-file
|
|
21
|
+
|
|
22
|
+
# sourced
|
|
23
|
+
|
|
24
|
+
[](https://github.com/MaXiMo000/sourced/actions/workflows/ci.yml)
|
|
25
|
+
|
|
26
|
+
**Checks an LLM's output against its own source context, claim by claim --
|
|
27
|
+
`grounded`, `contradicted`, or `unverified`, never one blended trust score.**
|
|
28
|
+
|
|
29
|
+
A RAG pipeline retrieves the right documents and still generates a sentence
|
|
30
|
+
those documents don't support -- a wrong number, an entity the source never
|
|
31
|
+
mentions, a stat close enough to sound plausible. Most RAG evaluation
|
|
32
|
+
scores *retrieval quality*: did the right chunks come back. Almost nothing
|
|
33
|
+
checks the *output* sentence by sentence against what was actually
|
|
34
|
+
retrieved. `sourced` does that second, narrower thing.
|
|
35
|
+
|
|
36
|
+
```
|
|
37
|
+
$ sourced check output.txt source.txt
|
|
38
|
+
[OK] Microsoft reported revenue of $56 billion.
|
|
39
|
+
[XX] Microsoft grew 40% year over year.
|
|
40
|
+
claims 40% near 'Microsoft', but the source's own text near that
|
|
41
|
+
entity says ['56', '8%']
|
|
42
|
+
[??] The outlook remains uncertain.
|
|
43
|
+
no checkable numbers, quoted text, or capitalized entities in this claim
|
|
44
|
+
|
|
45
|
+
1 grounded, 1 contradicted, 1 unverified
|
|
46
|
+
```
|
|
47
|
+
|
|
48
|
+
(That transcript is real output, not illustrative -- `source.txt` says growth
|
|
49
|
+
was 8%; note the second claim repeats "Microsoft" by name rather than
|
|
50
|
+
saying "it," because pronoun coreference isn't resolved -- see "What this
|
|
51
|
+
does NOT do.")
|
|
52
|
+
|
|
53
|
+
## Install
|
|
54
|
+
|
|
55
|
+
```
|
|
56
|
+
pip install sourced-evidence # the command it installs is `sourced`
|
|
57
|
+
```
|
|
58
|
+
|
|
59
|
+
(`sourced` was already taken on PyPI -- same story as `receipt-evidence`,
|
|
60
|
+
`providence-evidence`, and `custody-evidence` in this portfolio.)
|
|
61
|
+
|
|
62
|
+
## Use
|
|
63
|
+
|
|
64
|
+
```
|
|
65
|
+
sourced check <output-file> <source-file> [source-file ...] [--json]
|
|
66
|
+
```
|
|
67
|
+
|
|
68
|
+
`output-file` is the LLM's generated text. `source-file`(s) are the
|
|
69
|
+
context it was supposed to be grounded in -- the retrieved chunks, the
|
|
70
|
+
document it summarized, the transcript it's answering questions about.
|
|
71
|
+
Multiple source files are concatenated before checking.
|
|
72
|
+
|
|
73
|
+
Exit code is `1` only if any claim is `contradicted` -- same convention as
|
|
74
|
+
[`receipt`](https://github.com/MaXiMo000/receipt) and
|
|
75
|
+
[`invariant`](https://github.com/MaXiMo000/invariant): `unverified` is a
|
|
76
|
+
legitimate "can't tell," not a failure.
|
|
77
|
+
|
|
78
|
+
## How a claim gets checked
|
|
79
|
+
|
|
80
|
+
1. **Split into claims.** Every sentence in the output is one claim
|
|
81
|
+
candidate -- no attempt to tell a factual assertion from an opinion or
|
|
82
|
+
a hedge (`sourced/claims.py`).
|
|
83
|
+
2. **Extract signals.** Numbers, quoted substrings, and capitalized
|
|
84
|
+
entity-shaped phrases (`sourced/signals.py`) -- the concrete, matchable
|
|
85
|
+
facts a source text either does or doesn't contain.
|
|
86
|
+
3. **Classify against the source** (`sourced/check.py`):
|
|
87
|
+
- **`grounded`** -- every number, quote, and entity in the claim appears
|
|
88
|
+
in the source.
|
|
89
|
+
- **`contradicted`** -- a claim's number doesn't appear in the source at
|
|
90
|
+
all, but an entity from the *same claim* does, near a *different*
|
|
91
|
+
number. Narrow on purpose: this only fires when there's a real shared
|
|
92
|
+
anchor pinning the comparison to the same subject, never "two
|
|
93
|
+
different numbers exist somewhere in a long document."
|
|
94
|
+
- **`unverified`** -- everything else: no checkable signals in the claim
|
|
95
|
+
at all, or some signal simply isn't found anywhere in the source.
|
|
96
|
+
|
|
97
|
+
## What this does NOT do
|
|
98
|
+
|
|
99
|
+
This is the part worth reading before trusting a result.
|
|
100
|
+
|
|
101
|
+
- **No semantic understanding, no paraphrase matching.** "Revenue was $56
|
|
102
|
+
billion" and "the company made fifty-six billion dollars" are the same
|
|
103
|
+
fact and this will not see it that way -- it matches strings and
|
|
104
|
+
numbers, not meaning. A real semantic entailment checker needs a
|
|
105
|
+
language model; that's real future work (an optional LLM-backed
|
|
106
|
+
adjudication pass, the same shape `invariant`'s cascade or LabLedger's
|
|
107
|
+
Gemini stage already use elsewhere in this portfolio -- degrade
|
|
108
|
+
gracefully without it, don't require it), not built here.
|
|
109
|
+
- **No real named-entity recognition.** `signals.proper_nouns()` is a
|
|
110
|
+
capitalization heuristic, not a trained model. "Bank of America" splits
|
|
111
|
+
into `Bank` and `America` because a lowercase joiner breaks the
|
|
112
|
+
capitalized-word chain. Documented in `signals.py`, not hidden.
|
|
113
|
+
- **No real sentence tokenizer.** `claims.split_claims()` is a regex.
|
|
114
|
+
"Dr. Smith signed the report." reads as two sentences. Ordinary prose
|
|
115
|
+
splits correctly; abbreviation-heavy text won't always.
|
|
116
|
+
- **No pronoun or coreference resolution.** "Microsoft reported $56B. It
|
|
117
|
+
grew 40%." -- the second sentence's "It" is correctly excluded as an
|
|
118
|
+
entity (it's a pronoun, not a name), so there's no anchor to check that
|
|
119
|
+
claim's number against, and it comes back `unverified` rather than
|
|
120
|
+
`contradicted`. Each claim is checked entirely on its own; a real fix
|
|
121
|
+
needs coreference resolution, real NLP work of its own, not attempted
|
|
122
|
+
here.
|
|
123
|
+
- **Contradiction detection is deliberately conservative**, and that cuts
|
|
124
|
+
both ways: a real contradiction with no shared entity to anchor on comes
|
|
125
|
+
back `unverified`, not `contradicted` -- this will under-report
|
|
126
|
+
contradictions before it will over-report them. That's the intended
|
|
127
|
+
trade for a tool whose whole point is not manufacturing false
|
|
128
|
+
accusations against a source.
|
|
129
|
+
|
|
130
|
+
## Tests
|
|
131
|
+
|
|
132
|
+
```
|
|
133
|
+
pip install -e .
|
|
134
|
+
python tests/test_claims.py # sentence splitting
|
|
135
|
+
python tests/test_signals.py # number/quote/entity extraction
|
|
136
|
+
python tests/test_check.py # the grounded/contradicted/unverified decision
|
|
137
|
+
python tests/test_cli.py # the real CLI entry point, real files, real argv
|
|
138
|
+
```
|
|
139
|
+
|
|
140
|
+
41 tests. Several exist specifically because a first pass got something
|
|
141
|
+
wrong and testing against ordinary prose (not synthetic numbers-only
|
|
142
|
+
input) caught it -- e.g. a number regex that swallowed a following
|
|
143
|
+
sentence period (`"42."` parsed as the number `42.`), a sentence-
|
|
144
|
+
initial entity ("Microsoft" opening a sentence) needing case-insensitive
|
|
145
|
+
matching against a source that uses the same word lowercase mid-sentence,
|
|
146
|
+
without that leniency being used for anything else, and (found live
|
|
147
|
+
during a later audit) American-style closing punctuation putting the
|
|
148
|
+
period *inside* a quote (`"four hours."` not `"four hours".`) silently
|
|
149
|
+
merging two sentences into one claim -- the sentence-end regex only ever
|
|
150
|
+
checked for `[.!?]` immediately before the split point, never a quote
|
|
151
|
+
mark sitting in front of it.
|
|
152
|
+
|
|
153
|
+
MIT licensed.
|
|
@@ -0,0 +1,132 @@
|
|
|
1
|
+
# sourced
|
|
2
|
+
|
|
3
|
+
[](https://github.com/MaXiMo000/sourced/actions/workflows/ci.yml)
|
|
4
|
+
|
|
5
|
+
**Checks an LLM's output against its own source context, claim by claim --
|
|
6
|
+
`grounded`, `contradicted`, or `unverified`, never one blended trust score.**
|
|
7
|
+
|
|
8
|
+
A RAG pipeline retrieves the right documents and still generates a sentence
|
|
9
|
+
those documents don't support -- a wrong number, an entity the source never
|
|
10
|
+
mentions, a stat close enough to sound plausible. Most RAG evaluation
|
|
11
|
+
scores *retrieval quality*: did the right chunks come back. Almost nothing
|
|
12
|
+
checks the *output* sentence by sentence against what was actually
|
|
13
|
+
retrieved. `sourced` does that second, narrower thing.
|
|
14
|
+
|
|
15
|
+
```
|
|
16
|
+
$ sourced check output.txt source.txt
|
|
17
|
+
[OK] Microsoft reported revenue of $56 billion.
|
|
18
|
+
[XX] Microsoft grew 40% year over year.
|
|
19
|
+
claims 40% near 'Microsoft', but the source's own text near that
|
|
20
|
+
entity says ['56', '8%']
|
|
21
|
+
[??] The outlook remains uncertain.
|
|
22
|
+
no checkable numbers, quoted text, or capitalized entities in this claim
|
|
23
|
+
|
|
24
|
+
1 grounded, 1 contradicted, 1 unverified
|
|
25
|
+
```
|
|
26
|
+
|
|
27
|
+
(That transcript is real output, not illustrative -- `source.txt` says growth
|
|
28
|
+
was 8%; note the second claim repeats "Microsoft" by name rather than
|
|
29
|
+
saying "it," because pronoun coreference isn't resolved -- see "What this
|
|
30
|
+
does NOT do.")
|
|
31
|
+
|
|
32
|
+
## Install
|
|
33
|
+
|
|
34
|
+
```
|
|
35
|
+
pip install sourced-evidence # the command it installs is `sourced`
|
|
36
|
+
```
|
|
37
|
+
|
|
38
|
+
(`sourced` was already taken on PyPI -- same story as `receipt-evidence`,
|
|
39
|
+
`providence-evidence`, and `custody-evidence` in this portfolio.)
|
|
40
|
+
|
|
41
|
+
## Use
|
|
42
|
+
|
|
43
|
+
```
|
|
44
|
+
sourced check <output-file> <source-file> [source-file ...] [--json]
|
|
45
|
+
```
|
|
46
|
+
|
|
47
|
+
`output-file` is the LLM's generated text. `source-file`(s) are the
|
|
48
|
+
context it was supposed to be grounded in -- the retrieved chunks, the
|
|
49
|
+
document it summarized, the transcript it's answering questions about.
|
|
50
|
+
Multiple source files are concatenated before checking.
|
|
51
|
+
|
|
52
|
+
Exit code is `1` only if any claim is `contradicted` -- same convention as
|
|
53
|
+
[`receipt`](https://github.com/MaXiMo000/receipt) and
|
|
54
|
+
[`invariant`](https://github.com/MaXiMo000/invariant): `unverified` is a
|
|
55
|
+
legitimate "can't tell," not a failure.
|
|
56
|
+
|
|
57
|
+
## How a claim gets checked
|
|
58
|
+
|
|
59
|
+
1. **Split into claims.** Every sentence in the output is one claim
|
|
60
|
+
candidate -- no attempt to tell a factual assertion from an opinion or
|
|
61
|
+
a hedge (`sourced/claims.py`).
|
|
62
|
+
2. **Extract signals.** Numbers, quoted substrings, and capitalized
|
|
63
|
+
entity-shaped phrases (`sourced/signals.py`) -- the concrete, matchable
|
|
64
|
+
facts a source text either does or doesn't contain.
|
|
65
|
+
3. **Classify against the source** (`sourced/check.py`):
|
|
66
|
+
- **`grounded`** -- every number, quote, and entity in the claim appears
|
|
67
|
+
in the source.
|
|
68
|
+
- **`contradicted`** -- a claim's number doesn't appear in the source at
|
|
69
|
+
all, but an entity from the *same claim* does, near a *different*
|
|
70
|
+
number. Narrow on purpose: this only fires when there's a real shared
|
|
71
|
+
anchor pinning the comparison to the same subject, never "two
|
|
72
|
+
different numbers exist somewhere in a long document."
|
|
73
|
+
- **`unverified`** -- everything else: no checkable signals in the claim
|
|
74
|
+
at all, or some signal simply isn't found anywhere in the source.
|
|
75
|
+
|
|
76
|
+
## What this does NOT do
|
|
77
|
+
|
|
78
|
+
This is the part worth reading before trusting a result.
|
|
79
|
+
|
|
80
|
+
- **No semantic understanding, no paraphrase matching.** "Revenue was $56
|
|
81
|
+
billion" and "the company made fifty-six billion dollars" are the same
|
|
82
|
+
fact and this will not see it that way -- it matches strings and
|
|
83
|
+
numbers, not meaning. A real semantic entailment checker needs a
|
|
84
|
+
language model; that's real future work (an optional LLM-backed
|
|
85
|
+
adjudication pass, the same shape `invariant`'s cascade or LabLedger's
|
|
86
|
+
Gemini stage already use elsewhere in this portfolio -- degrade
|
|
87
|
+
gracefully without it, don't require it), not built here.
|
|
88
|
+
- **No real named-entity recognition.** `signals.proper_nouns()` is a
|
|
89
|
+
capitalization heuristic, not a trained model. "Bank of America" splits
|
|
90
|
+
into `Bank` and `America` because a lowercase joiner breaks the
|
|
91
|
+
capitalized-word chain. Documented in `signals.py`, not hidden.
|
|
92
|
+
- **No real sentence tokenizer.** `claims.split_claims()` is a regex.
|
|
93
|
+
"Dr. Smith signed the report." reads as two sentences. Ordinary prose
|
|
94
|
+
splits correctly; abbreviation-heavy text won't always.
|
|
95
|
+
- **No pronoun or coreference resolution.** "Microsoft reported $56B. It
|
|
96
|
+
grew 40%." -- the second sentence's "It" is correctly excluded as an
|
|
97
|
+
entity (it's a pronoun, not a name), so there's no anchor to check that
|
|
98
|
+
claim's number against, and it comes back `unverified` rather than
|
|
99
|
+
`contradicted`. Each claim is checked entirely on its own; a real fix
|
|
100
|
+
needs coreference resolution, real NLP work of its own, not attempted
|
|
101
|
+
here.
|
|
102
|
+
- **Contradiction detection is deliberately conservative**, and that cuts
|
|
103
|
+
both ways: a real contradiction with no shared entity to anchor on comes
|
|
104
|
+
back `unverified`, not `contradicted` -- this will under-report
|
|
105
|
+
contradictions before it will over-report them. That's the intended
|
|
106
|
+
trade for a tool whose whole point is not manufacturing false
|
|
107
|
+
accusations against a source.
|
|
108
|
+
|
|
109
|
+
## Tests
|
|
110
|
+
|
|
111
|
+
```
|
|
112
|
+
pip install -e .
|
|
113
|
+
python tests/test_claims.py # sentence splitting
|
|
114
|
+
python tests/test_signals.py # number/quote/entity extraction
|
|
115
|
+
python tests/test_check.py # the grounded/contradicted/unverified decision
|
|
116
|
+
python tests/test_cli.py # the real CLI entry point, real files, real argv
|
|
117
|
+
```
|
|
118
|
+
|
|
119
|
+
41 tests. Several exist specifically because a first pass got something
|
|
120
|
+
wrong and testing against ordinary prose (not synthetic numbers-only
|
|
121
|
+
input) caught it -- e.g. a number regex that swallowed a following
|
|
122
|
+
sentence period (`"42."` parsed as the number `42.`), a sentence-
|
|
123
|
+
initial entity ("Microsoft" opening a sentence) needing case-insensitive
|
|
124
|
+
matching against a source that uses the same word lowercase mid-sentence,
|
|
125
|
+
without that leniency being used for anything else, and (found live
|
|
126
|
+
during a later audit) American-style closing punctuation putting the
|
|
127
|
+
period *inside* a quote (`"four hours."` not `"four hours".`) silently
|
|
128
|
+
merging two sentences into one claim -- the sentence-end regex only ever
|
|
129
|
+
checked for `[.!?]` immediately before the split point, never a quote
|
|
130
|
+
mark sitting in front of it.
|
|
131
|
+
|
|
132
|
+
MIT licensed.
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
[project]
|
|
2
|
+
# "sourced" was already taken on PyPI, same story as receipt/providence/
|
|
3
|
+
# custody before it (checked, not assumed). The installed command stays
|
|
4
|
+
# the short name -- python-dateutil installs `dateutil`, this installs
|
|
5
|
+
# `sourced`.
|
|
6
|
+
name = "sourced-evidence"
|
|
7
|
+
version = "0.1.0"
|
|
8
|
+
description = "Per-claim grounded/contradicted/unverified checking of LLM output against its own source context -- never a blended trust score."
|
|
9
|
+
requires-python = ">=3.10"
|
|
10
|
+
readme = "README.md"
|
|
11
|
+
license = "MIT"
|
|
12
|
+
license-files = ["LICENSE"]
|
|
13
|
+
keywords = ["verification", "grounding", "llm", "rag", "evidence", "provenance"]
|
|
14
|
+
classifiers = [
|
|
15
|
+
"Development Status :: 3 - Alpha",
|
|
16
|
+
"Environment :: Console",
|
|
17
|
+
"Intended Audience :: Developers",
|
|
18
|
+
"Operating System :: OS Independent",
|
|
19
|
+
"Programming Language :: Python :: 3",
|
|
20
|
+
"Topic :: Software Development :: Testing",
|
|
21
|
+
]
|
|
22
|
+
# Stdlib only -- see README's "What this doesn't do": this is a lexical
|
|
23
|
+
# overlap checker, not a semantic one, on purpose, and needs no LLM API of
|
|
24
|
+
# its own to run.
|
|
25
|
+
dependencies = []
|
|
26
|
+
|
|
27
|
+
urls.Homepage = "https://github.com/MaXiMo000/sourced"
|
|
28
|
+
urls.Source = "https://github.com/MaXiMo000/sourced"
|
|
29
|
+
urls.Issues = "https://github.com/MaXiMo000/sourced/issues"
|
|
30
|
+
urls.Changelog = "https://github.com/MaXiMo000/sourced/releases"
|
|
31
|
+
|
|
32
|
+
[project.scripts]
|
|
33
|
+
sourced = "sourced.cli:main"
|
|
34
|
+
|
|
35
|
+
[build-system]
|
|
36
|
+
requires = ["setuptools>=77"]
|
|
37
|
+
build-backend = "setuptools.build_meta"
|
|
38
|
+
|
|
39
|
+
[tool.setuptools.packages.find]
|
|
40
|
+
include = ["sourced*"]
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
__version__ = "0.1.0"
|
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
"""Check one claim against source text: grounded, contradicted, or
|
|
2
|
+
unverified -- never a blended trust score. See README for exactly what
|
|
3
|
+
each status means, and what this deliberately does not attempt (semantic
|
|
4
|
+
entailment, paraphrase matching -- see "What this doesn't do").
|
|
5
|
+
"""
|
|
6
|
+
from __future__ import annotations
|
|
7
|
+
|
|
8
|
+
from . import signals
|
|
9
|
+
from .claims import split_claims
|
|
10
|
+
|
|
11
|
+
GROUNDED, CONTRADICTED, UNVERIFIED = "grounded", "contradicted", "unverified"
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def _window(text: str, index: int, radius: int = 60) -> str:
|
|
15
|
+
return text[max(0, index - radius): index + radius]
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def check_claim(claim: str, source: str) -> dict:
|
|
19
|
+
nums = signals.numbers(claim)
|
|
20
|
+
ents = signals.proper_nouns(claim)
|
|
21
|
+
quoted = signals.quotes(claim)
|
|
22
|
+
|
|
23
|
+
if not nums and not ents and not quoted:
|
|
24
|
+
return {
|
|
25
|
+
"claim": claim, "status": UNVERIFIED,
|
|
26
|
+
"detail": "no checkable numbers, quoted text, or capitalized entities in this claim",
|
|
27
|
+
"signals": {"numbers": [], "entities": [], "quotes": []},
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
# Contradiction: a number in the claim doesn't appear anywhere in the
|
|
31
|
+
# source, but an entity from the same claim does, near a *different*
|
|
32
|
+
# number. Narrow and conservative on purpose -- this only fires when
|
|
33
|
+
# there's a real anchor (a shared entity) pinning the comparison to the
|
|
34
|
+
# same subject, not "any two different numbers exist somewhere in a
|
|
35
|
+
# long document," which would be a false-contradiction machine.
|
|
36
|
+
contradictions = []
|
|
37
|
+
for ent in ents:
|
|
38
|
+
idx = source.find(ent)
|
|
39
|
+
if idx == -1:
|
|
40
|
+
continue
|
|
41
|
+
nearby_nums = signals.numbers(_window(source, idx))
|
|
42
|
+
for n in nums:
|
|
43
|
+
if n not in source and nearby_nums and n not in nearby_nums:
|
|
44
|
+
contradictions.append({
|
|
45
|
+
"claim_number": n, "entity": ent,
|
|
46
|
+
"source_numbers_nearby": nearby_nums,
|
|
47
|
+
})
|
|
48
|
+
|
|
49
|
+
if contradictions:
|
|
50
|
+
first = contradictions[0]
|
|
51
|
+
return {
|
|
52
|
+
"claim": claim, "status": CONTRADICTED,
|
|
53
|
+
"detail": (f"claims {first['claim_number']} near '{first['entity']}', but "
|
|
54
|
+
f"the source's own text near that entity says "
|
|
55
|
+
f"{first['source_numbers_nearby']}"),
|
|
56
|
+
"signals": {"numbers": nums, "entities": ents, "quotes": quoted},
|
|
57
|
+
"contradictions": contradictions,
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
missing_numbers = [n for n in nums if n not in source]
|
|
61
|
+
# Case-insensitive here, unlike the contradiction anchor above: a
|
|
62
|
+
# sentence-initial entity like "Revenue" is only capitalized because of
|
|
63
|
+
# its position, and the same word appears lowercase mid-sentence in
|
|
64
|
+
# most real source text ("Q3 revenue was..."). Exact-case matching
|
|
65
|
+
# would report a real match as "missing" for no reason other than
|
|
66
|
+
# capitalization -- a worse failure for a grounding check than being
|
|
67
|
+
# slightly lenient. Contradiction anchoring stays case-sensitive
|
|
68
|
+
# (source.find(ent) above) precisely because it needs to be stricter.
|
|
69
|
+
source_lower = source.lower()
|
|
70
|
+
missing_entities = [e for e in ents if e.lower() not in source_lower]
|
|
71
|
+
missing_quotes = [q for q in quoted if q not in source]
|
|
72
|
+
total = len(nums) + len(ents) + len(quoted)
|
|
73
|
+
missing = len(missing_numbers) + len(missing_entities) + len(missing_quotes)
|
|
74
|
+
|
|
75
|
+
if missing == 0:
|
|
76
|
+
return {
|
|
77
|
+
"claim": claim, "status": GROUNDED,
|
|
78
|
+
"detail": "every number, entity, and quoted phrase in this claim appears in the source",
|
|
79
|
+
"signals": {"numbers": nums, "entities": ents, "quotes": quoted},
|
|
80
|
+
}
|
|
81
|
+
return {
|
|
82
|
+
"claim": claim, "status": UNVERIFIED,
|
|
83
|
+
"detail": (f"{missing}/{total} signal(s) not found in the source -- "
|
|
84
|
+
f"numbers {missing_numbers}, entities {missing_entities}, "
|
|
85
|
+
f"quotes {missing_quotes}"),
|
|
86
|
+
"signals": {"numbers": nums, "entities": ents, "quotes": quoted},
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def check_output(output_text: str, source_text: str) -> dict:
|
|
91
|
+
"""Every sentence in `output_text`, checked against `source_text`."""
|
|
92
|
+
results = [check_claim(c, source_text) for c in split_claims(output_text)]
|
|
93
|
+
counts = {GROUNDED: 0, CONTRADICTED: 0, UNVERIFIED: 0}
|
|
94
|
+
for r in results:
|
|
95
|
+
counts[r["status"]] += 1
|
|
96
|
+
return {"claims": results, "counts": counts}
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
"""Split LLM output into claim candidates.
|
|
2
|
+
|
|
3
|
+
A "claim" here is just a sentence -- no attempt to distinguish a factual
|
|
4
|
+
assertion from an opinion, a question, or a hedge ("it seems that...").
|
|
5
|
+
That distinction is real NLP work of its own, and is explicitly out of
|
|
6
|
+
scope for v1 (see README's "What this doesn't do"). Treating every
|
|
7
|
+
sentence as a checkable claim means the tool never silently skips
|
|
8
|
+
something it should have checked, at the cost of also "checking" sentences
|
|
9
|
+
that were never factual assertions to begin with -- those simply come back
|
|
10
|
+
`unverified`, which is the correct, honest answer for a sentence with
|
|
11
|
+
nothing in it to check.
|
|
12
|
+
"""
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
import re
|
|
16
|
+
|
|
17
|
+
# Split on sentence-ending punctuation followed by whitespace and what looks
|
|
18
|
+
# like the start of a new sentence (a capital letter, a digit, or a quote).
|
|
19
|
+
# Deliberately a regex, not a real sentence tokenizer (spaCy/nltk): this
|
|
20
|
+
# portfolio stays dependency-free where a regex handles ordinary prose well
|
|
21
|
+
# enough. It will over-split or under-split on abbreviations ("Dr. Smith
|
|
22
|
+
# arrived.") and decimal numbers at a sentence boundary -- a stated
|
|
23
|
+
# limitation, not a silent one; see README.
|
|
24
|
+
#
|
|
25
|
+
# The lookbehind has two alternatives, not one optional closing-quote/paren
|
|
26
|
+
# character, because Python's re requires a fixed-width lookbehind: a
|
|
27
|
+
# quantifier inside (?<=...) is a SyntaxError. Both alternatives here are
|
|
28
|
+
# individually fixed-width (1 char, then 2 chars) -- the quantifier just
|
|
29
|
+
# isn't inside the lookbehind itself.
|
|
30
|
+
#
|
|
31
|
+
# The second alternative is the fix for a real bug: American-style closing
|
|
32
|
+
# punctuation puts the period *inside* the quote ("four hours." not "four
|
|
33
|
+
# hours".), so the character immediately before the split point is the
|
|
34
|
+
# quote mark, not [.!?] -- the single-alternative version never matched
|
|
35
|
+
# there at all, silently merging two sentences into one claim.
|
|
36
|
+
_SENTENCE_END = re.compile(r'(?:(?<=[.!?])|(?<=[.!?][\'")\]]))\s+(?=[A-Z0-9"\'])')
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def split_claims(text: str) -> list[str]:
|
|
40
|
+
text = text.strip()
|
|
41
|
+
if not text:
|
|
42
|
+
return []
|
|
43
|
+
parts = _SENTENCE_END.split(text)
|
|
44
|
+
return [p.strip() for p in parts if p.strip()]
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
"""sourced check <output-file> <source-file> [source-file ...] [--json]"""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
import argparse
|
|
5
|
+
import json
|
|
6
|
+
import pathlib
|
|
7
|
+
import sys
|
|
8
|
+
|
|
9
|
+
from .check import check_output
|
|
10
|
+
|
|
11
|
+
_TAG = {"grounded": "OK", "contradicted": "XX", "unverified": "??"}
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def _read(path: str) -> str:
|
|
15
|
+
"""A missing file, a directory given by mistake, or binary/undecodable
|
|
16
|
+
content is a wrong argument, not a crash -- every sibling tool in this
|
|
17
|
+
portfolio treats a bad file path as an actionable error message, not a
|
|
18
|
+
traceback (found by testing this against an actual typo'd path, the
|
|
19
|
+
same way providence's malformed-JSON crash was found)."""
|
|
20
|
+
try:
|
|
21
|
+
return pathlib.Path(path).read_text(encoding="utf-8")
|
|
22
|
+
except FileNotFoundError:
|
|
23
|
+
sys.exit(f"sourced: no such file: {path}")
|
|
24
|
+
except IsADirectoryError:
|
|
25
|
+
sys.exit(f"sourced: {path} is a directory, not a file")
|
|
26
|
+
except UnicodeDecodeError as exc:
|
|
27
|
+
sys.exit(f"sourced: {path} is not valid UTF-8 text ({exc})")
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def main(argv: list[str] | None = None) -> int:
|
|
31
|
+
parser = argparse.ArgumentParser(prog="sourced")
|
|
32
|
+
sub = parser.add_subparsers(dest="command", required=True)
|
|
33
|
+
|
|
34
|
+
check_p = sub.add_parser(
|
|
35
|
+
"check", help="check each sentence of an LLM output against source text")
|
|
36
|
+
check_p.add_argument("output_file", help="the LLM's output, one claim per sentence")
|
|
37
|
+
check_p.add_argument("source_files", nargs="+", help="the context it should be grounded in")
|
|
38
|
+
check_p.add_argument("--json", action="store_true", help="print the full report as JSON")
|
|
39
|
+
|
|
40
|
+
args = parser.parse_args(argv)
|
|
41
|
+
|
|
42
|
+
output_text = _read(args.output_file)
|
|
43
|
+
source_text = "\n".join(_read(f) for f in args.source_files)
|
|
44
|
+
report = check_output(output_text, source_text)
|
|
45
|
+
|
|
46
|
+
if args.json:
|
|
47
|
+
print(json.dumps(report, indent=2))
|
|
48
|
+
else:
|
|
49
|
+
for r in report["claims"]:
|
|
50
|
+
print(f"[{_TAG[r['status']]}] {r['claim']}")
|
|
51
|
+
if r["status"] != "grounded":
|
|
52
|
+
print(f" {r['detail']}")
|
|
53
|
+
c = report["counts"]
|
|
54
|
+
print(f"\n{c['grounded']} grounded, {c['contradicted']} contradicted, "
|
|
55
|
+
f"{c['unverified']} unverified")
|
|
56
|
+
|
|
57
|
+
# Same convention as receipt/invariant: unverified is a legitimate "we
|
|
58
|
+
# can't tell," not a failure -- only a contradiction (a claim actively
|
|
59
|
+
# at odds with its own source) is a nonzero exit.
|
|
60
|
+
return 1 if report["counts"]["contradicted"] > 0 else 0
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
if __name__ == "__main__":
|
|
64
|
+
raise SystemExit(main())
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
"""Extract the checkable content out of one claim: numbers, quoted
|
|
2
|
+
substrings, and proper-noun-shaped phrases. These are the concrete,
|
|
3
|
+
matchable facts a source text either does or doesn't contain -- the part
|
|
4
|
+
of "grounded vs unverified" that string matching can actually answer,
|
|
5
|
+
without needing to understand what the sentence means.
|
|
6
|
+
"""
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import re
|
|
10
|
+
|
|
11
|
+
# The fractional part requires a digit after the dot, not just an optional
|
|
12
|
+
# dot -- `\.?\d*` alone would swallow a following sentence period into the
|
|
13
|
+
# match ("42." at the end of a sentence reading as the number "42." instead
|
|
14
|
+
# of "42"), a real bug caught by testing this against ordinary prose, not
|
|
15
|
+
# synthetic numbers-only input.
|
|
16
|
+
_NUMBER = re.compile(r'-?\d[\d,]*(?:\.\d+)?%?')
|
|
17
|
+
_QUOTED = re.compile(r'"([^"]{3,})"|\'([^\']{3,})\'')
|
|
18
|
+
# A run of 1+ capitalized words -- a cheap proper-noun/entity proxy, not real
|
|
19
|
+
# named-entity recognition. Multi-word runs ("Bank of America" -- capitalized
|
|
20
|
+
# words joined by lowercase function words still read as one run since the
|
|
21
|
+
# regex only requires the *first* letter of each word to be uppercase and
|
|
22
|
+
# skips over up to one lowercase joiner... actually it doesn't: this simple
|
|
23
|
+
# version only chains directly-adjacent capitalized words. "Bank of America"
|
|
24
|
+
# would be seen as two separate entities, "Bank" and "America" -- a real,
|
|
25
|
+
# stated limitation of not doing real NER, not a bug to silently paper over.
|
|
26
|
+
_PROPER_NOUN = re.compile(r'\b[A-Z][a-zA-Z0-9]*(?:\s+[A-Z][a-zA-Z0-9]*)*\b')
|
|
27
|
+
|
|
28
|
+
# Common capitalized function words that are never an entity on their own
|
|
29
|
+
# (they're excluded even mid-sentence, e.g. "and The company" -- "The" is
|
|
30
|
+
# still not a proper noun there). This is the only filter single-word
|
|
31
|
+
# candidates get; see proper_nouns()'s docstring for why sentence position
|
|
32
|
+
# isn't used to filter them too.
|
|
33
|
+
_STOPWORD_CAPS = {
|
|
34
|
+
"The", "A", "An", "This", "That", "These", "Those", "It", "In", "On",
|
|
35
|
+
"At", "For", "As", "Is", "Was", "Were", "There", "Here", "But", "And",
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def numbers(text: str) -> list[str]:
|
|
40
|
+
return [m.group().replace(",", "") for m in _NUMBER.finditer(text)]
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def quotes(text: str) -> list[str]:
|
|
44
|
+
return [m.group(1) or m.group(2) for m in _QUOTED.finditer(text)]
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def proper_nouns(text: str) -> list[str]:
|
|
48
|
+
"""Multi-word capitalized runs are kept unconditionally (rarely a false
|
|
49
|
+
positive). A single capitalized word is kept unless it's a common
|
|
50
|
+
capitalized function word (the stoplist) -- deliberately NOT excluded
|
|
51
|
+
just for being sentence-initial: "Microsoft reported revenue of $70
|
|
52
|
+
billion" is exactly the shape this tool most needs to anchor a
|
|
53
|
+
contradiction check on, and Microsoft-as-subject is sentence-initial in
|
|
54
|
+
ordinary prose far more often than not. The real cost is the flip side:
|
|
55
|
+
an ordinary sentence-initial common noun ("Revenue grew...") gets
|
|
56
|
+
treated as a pseudo-entity too. That's an accepted, stated trade --
|
|
57
|
+
it only ever contributes to a *coverage* check ("Revenue" trivially
|
|
58
|
+
tends to appear in a source that's also about revenue), and it does
|
|
59
|
+
not on its own manufacture a contradiction the way a false anchor with
|
|
60
|
+
a genuinely different number nearby would.
|
|
61
|
+
"""
|
|
62
|
+
found = []
|
|
63
|
+
for m in _PROPER_NOUN.finditer(text):
|
|
64
|
+
phrase = m.group()
|
|
65
|
+
if " " in phrase or phrase not in _STOPWORD_CAPS:
|
|
66
|
+
found.append(phrase)
|
|
67
|
+
return found
|
|
@@ -0,0 +1,153 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: sourced-evidence
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Per-claim grounded/contradicted/unverified checking of LLM output against its own source context -- never a blended trust score.
|
|
5
|
+
License-Expression: MIT
|
|
6
|
+
Project-URL: Homepage, https://github.com/MaXiMo000/sourced
|
|
7
|
+
Project-URL: Source, https://github.com/MaXiMo000/sourced
|
|
8
|
+
Project-URL: Issues, https://github.com/MaXiMo000/sourced/issues
|
|
9
|
+
Project-URL: Changelog, https://github.com/MaXiMo000/sourced/releases
|
|
10
|
+
Keywords: verification,grounding,llm,rag,evidence,provenance
|
|
11
|
+
Classifier: Development Status :: 3 - Alpha
|
|
12
|
+
Classifier: Environment :: Console
|
|
13
|
+
Classifier: Intended Audience :: Developers
|
|
14
|
+
Classifier: Operating System :: OS Independent
|
|
15
|
+
Classifier: Programming Language :: Python :: 3
|
|
16
|
+
Classifier: Topic :: Software Development :: Testing
|
|
17
|
+
Requires-Python: >=3.10
|
|
18
|
+
Description-Content-Type: text/markdown
|
|
19
|
+
License-File: LICENSE
|
|
20
|
+
Dynamic: license-file
|
|
21
|
+
|
|
22
|
+
# sourced
|
|
23
|
+
|
|
24
|
+
[](https://github.com/MaXiMo000/sourced/actions/workflows/ci.yml)
|
|
25
|
+
|
|
26
|
+
**Checks an LLM's output against its own source context, claim by claim --
|
|
27
|
+
`grounded`, `contradicted`, or `unverified`, never one blended trust score.**
|
|
28
|
+
|
|
29
|
+
A RAG pipeline retrieves the right documents and still generates a sentence
|
|
30
|
+
those documents don't support -- a wrong number, an entity the source never
|
|
31
|
+
mentions, a stat close enough to sound plausible. Most RAG evaluation
|
|
32
|
+
scores *retrieval quality*: did the right chunks come back. Almost nothing
|
|
33
|
+
checks the *output* sentence by sentence against what was actually
|
|
34
|
+
retrieved. `sourced` does that second, narrower thing.
|
|
35
|
+
|
|
36
|
+
```
|
|
37
|
+
$ sourced check output.txt source.txt
|
|
38
|
+
[OK] Microsoft reported revenue of $56 billion.
|
|
39
|
+
[XX] Microsoft grew 40% year over year.
|
|
40
|
+
claims 40% near 'Microsoft', but the source's own text near that
|
|
41
|
+
entity says ['56', '8%']
|
|
42
|
+
[??] The outlook remains uncertain.
|
|
43
|
+
no checkable numbers, quoted text, or capitalized entities in this claim
|
|
44
|
+
|
|
45
|
+
1 grounded, 1 contradicted, 1 unverified
|
|
46
|
+
```
|
|
47
|
+
|
|
48
|
+
(That transcript is real output, not illustrative -- `source.txt` says growth
|
|
49
|
+
was 8%; note the second claim repeats "Microsoft" by name rather than
|
|
50
|
+
saying "it," because pronoun coreference isn't resolved -- see "What this
|
|
51
|
+
does NOT do.")
|
|
52
|
+
|
|
53
|
+
## Install
|
|
54
|
+
|
|
55
|
+
```
|
|
56
|
+
pip install sourced-evidence # the command it installs is `sourced`
|
|
57
|
+
```
|
|
58
|
+
|
|
59
|
+
(`sourced` was already taken on PyPI -- same story as `receipt-evidence`,
|
|
60
|
+
`providence-evidence`, and `custody-evidence` in this portfolio.)
|
|
61
|
+
|
|
62
|
+
## Use
|
|
63
|
+
|
|
64
|
+
```
|
|
65
|
+
sourced check <output-file> <source-file> [source-file ...] [--json]
|
|
66
|
+
```
|
|
67
|
+
|
|
68
|
+
`output-file` is the LLM's generated text. `source-file`(s) are the
|
|
69
|
+
context it was supposed to be grounded in -- the retrieved chunks, the
|
|
70
|
+
document it summarized, the transcript it's answering questions about.
|
|
71
|
+
Multiple source files are concatenated before checking.
|
|
72
|
+
|
|
73
|
+
Exit code is `1` only if any claim is `contradicted` -- same convention as
|
|
74
|
+
[`receipt`](https://github.com/MaXiMo000/receipt) and
|
|
75
|
+
[`invariant`](https://github.com/MaXiMo000/invariant): `unverified` is a
|
|
76
|
+
legitimate "can't tell," not a failure.
|
|
77
|
+
|
|
78
|
+
## How a claim gets checked
|
|
79
|
+
|
|
80
|
+
1. **Split into claims.** Every sentence in the output is one claim
|
|
81
|
+
candidate -- no attempt to tell a factual assertion from an opinion or
|
|
82
|
+
a hedge (`sourced/claims.py`).
|
|
83
|
+
2. **Extract signals.** Numbers, quoted substrings, and capitalized
|
|
84
|
+
entity-shaped phrases (`sourced/signals.py`) -- the concrete, matchable
|
|
85
|
+
facts a source text either does or doesn't contain.
|
|
86
|
+
3. **Classify against the source** (`sourced/check.py`):
|
|
87
|
+
- **`grounded`** -- every number, quote, and entity in the claim appears
|
|
88
|
+
in the source.
|
|
89
|
+
- **`contradicted`** -- a claim's number doesn't appear in the source at
|
|
90
|
+
all, but an entity from the *same claim* does, near a *different*
|
|
91
|
+
number. Narrow on purpose: this only fires when there's a real shared
|
|
92
|
+
anchor pinning the comparison to the same subject, never "two
|
|
93
|
+
different numbers exist somewhere in a long document."
|
|
94
|
+
- **`unverified`** -- everything else: no checkable signals in the claim
|
|
95
|
+
at all, or some signal simply isn't found anywhere in the source.
|
|
96
|
+
|
|
97
|
+
## What this does NOT do
|
|
98
|
+
|
|
99
|
+
This is the part worth reading before trusting a result.
|
|
100
|
+
|
|
101
|
+
- **No semantic understanding, no paraphrase matching.** "Revenue was $56
|
|
102
|
+
billion" and "the company made fifty-six billion dollars" are the same
|
|
103
|
+
fact and this will not see it that way -- it matches strings and
|
|
104
|
+
numbers, not meaning. A real semantic entailment checker needs a
|
|
105
|
+
language model; that's real future work (an optional LLM-backed
|
|
106
|
+
adjudication pass, the same shape `invariant`'s cascade or LabLedger's
|
|
107
|
+
Gemini stage already use elsewhere in this portfolio -- degrade
|
|
108
|
+
gracefully without it, don't require it), not built here.
|
|
109
|
+
- **No real named-entity recognition.** `signals.proper_nouns()` is a
|
|
110
|
+
capitalization heuristic, not a trained model. "Bank of America" splits
|
|
111
|
+
into `Bank` and `America` because a lowercase joiner breaks the
|
|
112
|
+
capitalized-word chain. Documented in `signals.py`, not hidden.
|
|
113
|
+
- **No real sentence tokenizer.** `claims.split_claims()` is a regex.
|
|
114
|
+
"Dr. Smith signed the report." reads as two sentences. Ordinary prose
|
|
115
|
+
splits correctly; abbreviation-heavy text won't always.
|
|
116
|
+
- **No pronoun or coreference resolution.** "Microsoft reported $56B. It
|
|
117
|
+
grew 40%." -- the second sentence's "It" is correctly excluded as an
|
|
118
|
+
entity (it's a pronoun, not a name), so there's no anchor to check that
|
|
119
|
+
claim's number against, and it comes back `unverified` rather than
|
|
120
|
+
`contradicted`. Each claim is checked entirely on its own; a real fix
|
|
121
|
+
needs coreference resolution, real NLP work of its own, not attempted
|
|
122
|
+
here.
|
|
123
|
+
- **Contradiction detection is deliberately conservative**, and that cuts
|
|
124
|
+
both ways: a real contradiction with no shared entity to anchor on comes
|
|
125
|
+
back `unverified`, not `contradicted` -- this will under-report
|
|
126
|
+
contradictions before it will over-report them. That's the intended
|
|
127
|
+
trade for a tool whose whole point is not manufacturing false
|
|
128
|
+
accusations against a source.
|
|
129
|
+
|
|
130
|
+
## Tests
|
|
131
|
+
|
|
132
|
+
```
|
|
133
|
+
pip install -e .
|
|
134
|
+
python tests/test_claims.py # sentence splitting
|
|
135
|
+
python tests/test_signals.py # number/quote/entity extraction
|
|
136
|
+
python tests/test_check.py # the grounded/contradicted/unverified decision
|
|
137
|
+
python tests/test_cli.py # the real CLI entry point, real files, real argv
|
|
138
|
+
```
|
|
139
|
+
|
|
140
|
+
41 tests. Several exist specifically because a first pass got something
|
|
141
|
+
wrong and testing against ordinary prose (not synthetic numbers-only
|
|
142
|
+
input) caught it -- e.g. a number regex that swallowed a following
|
|
143
|
+
sentence period (`"42."` parsed as the number `42.`), a sentence-
|
|
144
|
+
initial entity ("Microsoft" opening a sentence) needing case-insensitive
|
|
145
|
+
matching against a source that uses the same word lowercase mid-sentence,
|
|
146
|
+
without that leniency being used for anything else, and (found live
|
|
147
|
+
during a later audit) American-style closing punctuation putting the
|
|
148
|
+
period *inside* a quote (`"four hours."` not `"four hours".`) silently
|
|
149
|
+
merging two sentences into one claim -- the sentence-end regex only ever
|
|
150
|
+
checked for `[.!?]` immediately before the split point, never a quote
|
|
151
|
+
mark sitting in front of it.
|
|
152
|
+
|
|
153
|
+
MIT licensed.
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
LICENSE
|
|
2
|
+
README.md
|
|
3
|
+
pyproject.toml
|
|
4
|
+
sourced/__init__.py
|
|
5
|
+
sourced/check.py
|
|
6
|
+
sourced/claims.py
|
|
7
|
+
sourced/cli.py
|
|
8
|
+
sourced/signals.py
|
|
9
|
+
sourced_evidence.egg-info/PKG-INFO
|
|
10
|
+
sourced_evidence.egg-info/SOURCES.txt
|
|
11
|
+
sourced_evidence.egg-info/dependency_links.txt
|
|
12
|
+
sourced_evidence.egg-info/entry_points.txt
|
|
13
|
+
sourced_evidence.egg-info/top_level.txt
|
|
14
|
+
tests/test_check.py
|
|
15
|
+
tests/test_claims.py
|
|
16
|
+
tests/test_cli.py
|
|
17
|
+
tests/test_signals.py
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
sourced
|
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
"""Run: python tests/test_check.py"""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
import pathlib
|
|
5
|
+
import sys
|
|
6
|
+
import unittest
|
|
7
|
+
|
|
8
|
+
sys.path.insert(0, str(pathlib.Path(__file__).parent.parent))
|
|
9
|
+
|
|
10
|
+
from sourced.check import CONTRADICTED, GROUNDED, UNVERIFIED, check_claim, check_output
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class TestCheckClaim(unittest.TestCase):
|
|
14
|
+
def test_claim_with_no_signals_is_unverified(self):
|
|
15
|
+
r = check_claim("The company grew nicely this quarter.", "Some unrelated source text.")
|
|
16
|
+
self.assertEqual(r["status"], UNVERIFIED)
|
|
17
|
+
self.assertIn("no checkable", r["detail"])
|
|
18
|
+
|
|
19
|
+
def test_number_found_verbatim_in_source_is_grounded(self):
|
|
20
|
+
"""Also the entity-case-insensitivity regression test: "Revenue" is
|
|
21
|
+
a pseudo-entity purely because it's sentence-initial, and the
|
|
22
|
+
source only has it lowercase ("Q3 revenue was..."). Case-sensitive
|
|
23
|
+
entity matching used to report this real match as "missing" for no
|
|
24
|
+
reason other than capitalization, dragging an otherwise-grounded
|
|
25
|
+
claim down to unverified."""
|
|
26
|
+
source = "Q3 revenue was $42 million, up from $38 million a year ago."
|
|
27
|
+
r = check_claim("Revenue was $42 million.", source)
|
|
28
|
+
self.assertEqual(r["status"], GROUNDED)
|
|
29
|
+
|
|
30
|
+
def test_number_absent_from_source_with_no_anchor_is_unverified_not_contradicted(self):
|
|
31
|
+
"""No shared entity to pin the comparison on -- this can only say
|
|
32
|
+
'not found', not 'the source disagrees'."""
|
|
33
|
+
source = "The company had a strong quarter overall."
|
|
34
|
+
r = check_claim("Revenue was $42 million.", source)
|
|
35
|
+
self.assertEqual(r["status"], UNVERIFIED)
|
|
36
|
+
|
|
37
|
+
def test_entity_present_with_a_different_number_nearby_is_contradicted(self):
|
|
38
|
+
source = "Microsoft reported revenue of $56 billion for the quarter."
|
|
39
|
+
r = check_claim("Microsoft reported revenue of $70 billion.", source)
|
|
40
|
+
self.assertEqual(r["status"], CONTRADICTED)
|
|
41
|
+
self.assertEqual(r["contradictions"][0]["entity"], "Microsoft")
|
|
42
|
+
self.assertIn("56", r["contradictions"][0]["source_numbers_nearby"])
|
|
43
|
+
|
|
44
|
+
def test_entity_present_with_the_same_number_is_grounded_not_contradicted(self):
|
|
45
|
+
source = "Microsoft reported revenue of $56 billion for the quarter."
|
|
46
|
+
r = check_claim("Microsoft reported revenue of $56 billion.", source)
|
|
47
|
+
self.assertEqual(r["status"], GROUNDED)
|
|
48
|
+
|
|
49
|
+
def test_quoted_phrase_present_verbatim_is_grounded(self):
|
|
50
|
+
source = 'The CEO said "we exceeded every target this year."'
|
|
51
|
+
r = check_claim('The CEO said "we exceeded every target this year."', source)
|
|
52
|
+
self.assertEqual(r["status"], GROUNDED)
|
|
53
|
+
|
|
54
|
+
def test_quoted_phrase_not_in_source_is_unverified(self):
|
|
55
|
+
source = "The CEO discussed the results at length."
|
|
56
|
+
r = check_claim('The CEO said "this was our best year ever."', source)
|
|
57
|
+
self.assertEqual(r["status"], UNVERIFIED)
|
|
58
|
+
self.assertIn("this was our best year ever", r["detail"])
|
|
59
|
+
|
|
60
|
+
def test_entity_only_partially_covered_is_unverified(self):
|
|
61
|
+
source = "Apple's revenue grew steadily."
|
|
62
|
+
r = check_claim("Apple and Microsoft both grew revenue.", source)
|
|
63
|
+
self.assertEqual(r["status"], UNVERIFIED)
|
|
64
|
+
self.assertIn("Microsoft", r["detail"])
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
class TestCheckOutput(unittest.TestCase):
|
|
68
|
+
def test_multiple_claims_are_each_checked_independently(self):
|
|
69
|
+
source = "Microsoft reported revenue of $56 billion. Apple grew 8%."
|
|
70
|
+
output = "Microsoft reported revenue of $56 billion. Apple grew 40%. The mood was upbeat."
|
|
71
|
+
report = check_output(output, source)
|
|
72
|
+
self.assertEqual(len(report["claims"]), 3)
|
|
73
|
+
statuses = [c["status"] for c in report["claims"]]
|
|
74
|
+
self.assertEqual(statuses, [GROUNDED, CONTRADICTED, UNVERIFIED])
|
|
75
|
+
self.assertEqual(report["counts"], {"grounded": 1, "contradicted": 1, "unverified": 1})
|
|
76
|
+
|
|
77
|
+
def test_empty_output_is_zero_claims_not_an_error(self):
|
|
78
|
+
report = check_output("", "some source")
|
|
79
|
+
self.assertEqual(report["claims"], [])
|
|
80
|
+
self.assertEqual(report["counts"], {"grounded": 0, "contradicted": 0, "unverified": 0})
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
if __name__ == "__main__":
|
|
84
|
+
unittest.main()
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
"""Run: python tests/test_claims.py"""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
import pathlib
|
|
5
|
+
import sys
|
|
6
|
+
import unittest
|
|
7
|
+
|
|
8
|
+
sys.path.insert(0, str(pathlib.Path(__file__).parent.parent))
|
|
9
|
+
|
|
10
|
+
from sourced.claims import split_claims
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class TestSplitClaims(unittest.TestCase):
|
|
14
|
+
def test_empty_text_is_no_claims(self):
|
|
15
|
+
self.assertEqual(split_claims(""), [])
|
|
16
|
+
self.assertEqual(split_claims(" "), [])
|
|
17
|
+
|
|
18
|
+
def test_single_sentence_is_one_claim(self):
|
|
19
|
+
self.assertEqual(split_claims("Revenue grew 12% in Q3."),
|
|
20
|
+
["Revenue grew 12% in Q3."])
|
|
21
|
+
|
|
22
|
+
def test_multiple_sentences_split_on_terminal_punctuation(self):
|
|
23
|
+
text = "Revenue grew 12% in Q3. Costs fell slightly. Margins improved."
|
|
24
|
+
self.assertEqual(split_claims(text), [
|
|
25
|
+
"Revenue grew 12% in Q3.",
|
|
26
|
+
"Costs fell slightly.",
|
|
27
|
+
"Margins improved.",
|
|
28
|
+
])
|
|
29
|
+
|
|
30
|
+
def test_question_marks_and_exclamations_also_split(self):
|
|
31
|
+
text = "Did revenue grow? Yes, by 12%! That beat expectations."
|
|
32
|
+
self.assertEqual(len(split_claims(text)), 3)
|
|
33
|
+
|
|
34
|
+
def test_a_decimal_number_does_not_split_mid_sentence(self):
|
|
35
|
+
"""3.14 is not two sentences -- the split only fires when a capital,
|
|
36
|
+
digit, or quote follows the punctuation+space, and a lowercase
|
|
37
|
+
continuation (or no space at all, as in a decimal) doesn't count."""
|
|
38
|
+
text = "Pi is approximately 3.14 and shows up everywhere."
|
|
39
|
+
self.assertEqual(split_claims(text), [text])
|
|
40
|
+
|
|
41
|
+
def test_known_limitation_abbreviations_over_split(self):
|
|
42
|
+
"""Documented, not hidden: 'Dr. Smith' reads as two sentences,
|
|
43
|
+
since this is a regex, not a real tokenizer that knows abbreviations."""
|
|
44
|
+
text = "Dr. Smith signed the report."
|
|
45
|
+
self.assertEqual(len(split_claims(text)), 2)
|
|
46
|
+
|
|
47
|
+
def test_a_period_inside_a_closing_quote_still_splits(self):
|
|
48
|
+
"""Real bug, found live: American-style punctuation puts the period
|
|
49
|
+
*inside* the quote ("four hours." not "four hours".), so the
|
|
50
|
+
character right before the split point is the quote mark, not
|
|
51
|
+
[.!?] -- the original lookbehind only ever checked for [.!?]
|
|
52
|
+
directly, so this silently merged two claims into one."""
|
|
53
|
+
text = 'The report said the outage lasted "four hours." The team resolved it by noon.'
|
|
54
|
+
self.assertEqual(split_claims(text), [
|
|
55
|
+
'The report said the outage lasted "four hours."',
|
|
56
|
+
"The team resolved it by noon.",
|
|
57
|
+
])
|
|
58
|
+
|
|
59
|
+
def test_a_period_outside_a_closing_quote_still_splits(self):
|
|
60
|
+
"""The other quoting convention already worked (the period itself
|
|
61
|
+
sits right before the space) -- pinned so a future regex change
|
|
62
|
+
can't fix one convention by breaking the other."""
|
|
63
|
+
text = 'The report said the outage lasted "four hours". The team resolved it by noon.'
|
|
64
|
+
self.assertEqual(split_claims(text), [
|
|
65
|
+
'The report said the outage lasted "four hours".',
|
|
66
|
+
"The team resolved it by noon.",
|
|
67
|
+
])
|
|
68
|
+
|
|
69
|
+
def test_a_period_inside_a_closing_paren_still_splits(self):
|
|
70
|
+
text = "She said it works (allegedly.) Then it broke."
|
|
71
|
+
self.assertEqual(split_claims(text), [
|
|
72
|
+
"She said it works (allegedly.)",
|
|
73
|
+
"Then it broke.",
|
|
74
|
+
])
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
if __name__ == "__main__":
|
|
78
|
+
unittest.main()
|
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
"""Run: python tests/test_cli.py
|
|
2
|
+
|
|
3
|
+
Exercises the real CLI entry point end to end -- real temp files, real
|
|
4
|
+
argv, real stdout capture -- not just the pure check_output() function
|
|
5
|
+
underneath it.
|
|
6
|
+
"""
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import contextlib
|
|
10
|
+
import io
|
|
11
|
+
import json
|
|
12
|
+
import pathlib
|
|
13
|
+
import sys
|
|
14
|
+
import tempfile
|
|
15
|
+
import unittest
|
|
16
|
+
|
|
17
|
+
sys.path.insert(0, str(pathlib.Path(__file__).parent.parent))
|
|
18
|
+
|
|
19
|
+
from sourced.cli import main
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class TestCli(unittest.TestCase):
|
|
23
|
+
def setUp(self):
|
|
24
|
+
self.tmp = tempfile.TemporaryDirectory()
|
|
25
|
+
self.dir = pathlib.Path(self.tmp.name)
|
|
26
|
+
|
|
27
|
+
def tearDown(self):
|
|
28
|
+
self.tmp.cleanup()
|
|
29
|
+
|
|
30
|
+
def _write(self, name: str, content: str) -> str:
|
|
31
|
+
p = self.dir / name
|
|
32
|
+
p.write_text(content, encoding="utf-8")
|
|
33
|
+
return str(p)
|
|
34
|
+
|
|
35
|
+
def test_exit_zero_when_nothing_contradicted(self):
|
|
36
|
+
output = self._write("output.txt", "Microsoft reported revenue of $56 billion.")
|
|
37
|
+
source = self._write("source.txt", "Microsoft reported revenue of $56 billion.")
|
|
38
|
+
buf = io.StringIO()
|
|
39
|
+
with contextlib.redirect_stdout(buf):
|
|
40
|
+
code = main(["check", output, source])
|
|
41
|
+
self.assertEqual(code, 0)
|
|
42
|
+
self.assertIn("1 grounded", buf.getvalue())
|
|
43
|
+
|
|
44
|
+
def test_exit_one_when_a_claim_is_contradicted(self):
|
|
45
|
+
output = self._write("output.txt", "Microsoft reported revenue of $70 billion.")
|
|
46
|
+
source = self._write("source.txt", "Microsoft reported revenue of $56 billion.")
|
|
47
|
+
buf = io.StringIO()
|
|
48
|
+
with contextlib.redirect_stdout(buf):
|
|
49
|
+
code = main(["check", output, source])
|
|
50
|
+
self.assertEqual(code, 1)
|
|
51
|
+
self.assertIn("1 contradicted", buf.getvalue())
|
|
52
|
+
|
|
53
|
+
def test_multiple_source_files_are_concatenated(self):
|
|
54
|
+
output = self._write("output.txt", "Microsoft reported revenue of $56 billion.")
|
|
55
|
+
s1 = self._write("s1.txt", "Some unrelated background.")
|
|
56
|
+
s2 = self._write("s2.txt", "Microsoft reported revenue of $56 billion.")
|
|
57
|
+
buf = io.StringIO()
|
|
58
|
+
with contextlib.redirect_stdout(buf):
|
|
59
|
+
code = main(["check", output, s1, s2])
|
|
60
|
+
self.assertEqual(code, 0)
|
|
61
|
+
self.assertIn("1 grounded", buf.getvalue())
|
|
62
|
+
|
|
63
|
+
def test_missing_source_file_is_a_clean_error_not_a_traceback(self):
|
|
64
|
+
"""Found by testing an actual typo'd path, the same way
|
|
65
|
+
providence's malformed-JSON crash was found: a missing file used
|
|
66
|
+
to raise FileNotFoundError straight out of main()."""
|
|
67
|
+
output = self._write("output.txt", "A claim.")
|
|
68
|
+
with self.assertRaises(SystemExit) as ctx:
|
|
69
|
+
main(["check", output, str(self.dir / "nope.txt")])
|
|
70
|
+
self.assertIn("no such file", str(ctx.exception))
|
|
71
|
+
|
|
72
|
+
def test_a_directory_given_instead_of_a_file_is_a_clean_error(self):
|
|
73
|
+
output = self._write("output.txt", "A claim.")
|
|
74
|
+
with self.assertRaises(SystemExit) as ctx:
|
|
75
|
+
main(["check", output, str(self.dir)])
|
|
76
|
+
self.assertIn("is a directory", str(ctx.exception))
|
|
77
|
+
|
|
78
|
+
def test_json_flag_prints_the_full_report(self):
|
|
79
|
+
output = self._write("output.txt", "Microsoft reported revenue of $56 billion.")
|
|
80
|
+
source = self._write("source.txt", "Microsoft reported revenue of $56 billion.")
|
|
81
|
+
buf = io.StringIO()
|
|
82
|
+
with contextlib.redirect_stdout(buf):
|
|
83
|
+
main(["check", output, source, "--json"])
|
|
84
|
+
report = json.loads(buf.getvalue())
|
|
85
|
+
self.assertEqual(report["counts"]["grounded"], 1)
|
|
86
|
+
self.assertEqual(report["claims"][0]["status"], "grounded")
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
if __name__ == "__main__":
|
|
90
|
+
unittest.main()
|
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
"""Run: python tests/test_signals.py"""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
import pathlib
|
|
5
|
+
import sys
|
|
6
|
+
import unittest
|
|
7
|
+
|
|
8
|
+
sys.path.insert(0, str(pathlib.Path(__file__).parent.parent))
|
|
9
|
+
|
|
10
|
+
from sourced import signals
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class TestNumbers(unittest.TestCase):
|
|
14
|
+
def test_plain_integer(self):
|
|
15
|
+
self.assertEqual(signals.numbers("Revenue was 42."), ["42"])
|
|
16
|
+
|
|
17
|
+
def test_decimal_and_percent(self):
|
|
18
|
+
self.assertEqual(signals.numbers("Grew 12.5% this quarter."), ["12.5%"])
|
|
19
|
+
|
|
20
|
+
def test_comma_thousands_separator_is_normalized(self):
|
|
21
|
+
self.assertEqual(signals.numbers("Sold 1,200,000 units."), ["1200000"])
|
|
22
|
+
|
|
23
|
+
def test_negative_number(self):
|
|
24
|
+
self.assertEqual(signals.numbers("Margin fell -3.2%."), ["-3.2%"])
|
|
25
|
+
|
|
26
|
+
def test_no_numbers(self):
|
|
27
|
+
self.assertEqual(signals.numbers("Revenue grew nicely."), [])
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
class TestQuotes(unittest.TestCase):
|
|
31
|
+
def test_double_quoted_phrase(self):
|
|
32
|
+
self.assertEqual(signals.quotes('The CEO called it "a record quarter."'),
|
|
33
|
+
["a record quarter."])
|
|
34
|
+
|
|
35
|
+
def test_single_quoted_phrase(self):
|
|
36
|
+
self.assertEqual(signals.quotes("She called it 'a turning point'."),
|
|
37
|
+
["a turning point"])
|
|
38
|
+
|
|
39
|
+
def test_short_quotes_under_three_chars_are_ignored(self):
|
|
40
|
+
"""A stray apostrophe pair around one letter isn't a quoted claim
|
|
41
|
+
worth checking -- e.g. a possessive or a contraction fragment."""
|
|
42
|
+
self.assertEqual(signals.quotes("It's 'a' small win."), [])
|
|
43
|
+
|
|
44
|
+
def test_no_quotes(self):
|
|
45
|
+
self.assertEqual(signals.quotes("Revenue grew 12%."), [])
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
class TestProperNouns(unittest.TestCase):
|
|
49
|
+
def test_lowercase_joiner_splits_a_real_multiword_entity(self):
|
|
50
|
+
"""Documented limitation, not a bug: real NER would see one entity,
|
|
51
|
+
'Bank of America'. This regex only chains directly-adjacent
|
|
52
|
+
capitalized words, so 'of' breaks the chain -- 'Bank' (sentence-
|
|
53
|
+
initial, so excluded) and 'America' (kept) come out separately."""
|
|
54
|
+
self.assertEqual(signals.proper_nouns("Bank of America reported earnings."),
|
|
55
|
+
["Bank", "America"])
|
|
56
|
+
|
|
57
|
+
def test_adjacent_capitalized_words_do_chain(self):
|
|
58
|
+
self.assertEqual(signals.proper_nouns("It involved New York City directly."),
|
|
59
|
+
["New York City"])
|
|
60
|
+
|
|
61
|
+
def test_sentence_initial_subject_is_kept_as_a_candidate_entity(self):
|
|
62
|
+
"""Deliberate, not a miss: 'Microsoft reported X' is exactly the
|
|
63
|
+
subject-first shape a contradiction check needs to anchor on, and
|
|
64
|
+
excluding anything sentence-initial would silently drop it. The
|
|
65
|
+
real cost -- an ordinary sentence-initial common noun like
|
|
66
|
+
'Revenue' also becomes a pseudo-entity -- is accepted and
|
|
67
|
+
documented in signals.py, not hidden."""
|
|
68
|
+
self.assertEqual(signals.proper_nouns("Revenue grew 12% in Q3."), ["Revenue", "Q3"])
|
|
69
|
+
|
|
70
|
+
def test_stopword_caps_excluded_even_sentence_initial(self):
|
|
71
|
+
self.assertEqual(signals.proper_nouns("The company grew 12% in Q3."), ["Q3"])
|
|
72
|
+
|
|
73
|
+
def test_midsentence_single_capitalized_word_is_kept(self):
|
|
74
|
+
self.assertIn("Microsoft", signals.proper_nouns("The deal involved Microsoft directly."))
|
|
75
|
+
|
|
76
|
+
def test_stopword_caps_excluded_even_midsentence(self):
|
|
77
|
+
self.assertNotIn("The", signals.proper_nouns("It grew, and The company noted it."))
|
|
78
|
+
|
|
79
|
+
def test_no_entities(self):
|
|
80
|
+
self.assertEqual(signals.proper_nouns("revenue grew twelve percent"), [])
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
if __name__ == "__main__":
|
|
84
|
+
unittest.main()
|