k-uncanny 0.1.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- k_uncanny-0.1.1/PKG-INFO +223 -0
- k_uncanny-0.1.1/README.md +195 -0
- k_uncanny-0.1.1/pyproject.toml +45 -0
- k_uncanny-0.1.1/src/uncanny/__init__.py +5 -0
- k_uncanny-0.1.1/src/uncanny/__init__.py~ +3 -0
- k_uncanny-0.1.1/src/uncanny/__main__.py +3 -0
- k_uncanny-0.1.1/src/uncanny/analyzers/__init__.py +13 -0
- k_uncanny-0.1.1/src/uncanny/analyzers/base.py +16 -0
- k_uncanny-0.1.1/src/uncanny/analyzers/burstiness.py +121 -0
- k_uncanny-0.1.1/src/uncanny/analyzers/compression.py +180 -0
- k_uncanny-0.1.1/src/uncanny/analyzers/perplexity.py +99 -0
- k_uncanny-0.1.1/src/uncanny/bench/__init__.py +0 -0
- k_uncanny-0.1.1/src/uncanny/bench/competitors.py +166 -0
- k_uncanny-0.1.1/src/uncanny/bench/display.py +148 -0
- k_uncanny-0.1.1/src/uncanny/bench/runner.py +276 -0
- k_uncanny-0.1.1/src/uncanny/cli.py +251 -0
- k_uncanny-0.1.1/src/uncanny/ensemble.py +105 -0
- k_uncanny-0.1.1/src/uncanny/output/__init__.py +0 -0
- k_uncanny-0.1.1/src/uncanny/output/json_output.py +12 -0
- k_uncanny-0.1.1/src/uncanny/output/terminal.py +122 -0
- k_uncanny-0.1.1/src/uncanny/py.typed +0 -0
- k_uncanny-0.1.1/src/uncanny/tokenizer.py +60 -0
- k_uncanny-0.1.1/src/uncanny/types.py +50 -0
- k_uncanny-0.1.1/src/uncanny/web/__init__.py +0 -0
- k_uncanny-0.1.1/src/uncanny/web/server.py +96 -0
- k_uncanny-0.1.1/src/uncanny/web/static/index.html +564 -0
k_uncanny-0.1.1/PKG-INFO
ADDED
|
@@ -0,0 +1,223 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: k-uncanny
|
|
3
|
+
Version: 0.1.1
|
|
4
|
+
Summary: CLI tool that detects AI-generated text with sentence-level scoring and multiple detection strategies
|
|
5
|
+
Keywords: ai-detection,gpt,llm,text-analysis,cli
|
|
6
|
+
Author: Frank Chan
|
|
7
|
+
License-Expression: MIT
|
|
8
|
+
Classifier: Development Status :: 3 - Alpha
|
|
9
|
+
Classifier: Environment :: Console
|
|
10
|
+
Classifier: Intended Audience :: Developers
|
|
11
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
12
|
+
Classifier: Programming Language :: Python :: 3
|
|
13
|
+
Classifier: Topic :: Text Processing :: Linguistic
|
|
14
|
+
Requires-Dist: typer>=0.9
|
|
15
|
+
Requires-Dist: rich>=13.0
|
|
16
|
+
Requires-Dist: pytest>=7.0 ; extra == 'dev'
|
|
17
|
+
Requires-Dist: pytest-cov>=4.0 ; extra == 'dev'
|
|
18
|
+
Requires-Dist: torch>=2.0 ; extra == 'ml'
|
|
19
|
+
Requires-Dist: transformers>=4.30 ; extra == 'ml'
|
|
20
|
+
Requires-Python: >=3.10
|
|
21
|
+
Project-URL: Homepage, https://github.com/kpihx-labs/uncanny
|
|
22
|
+
Project-URL: Repository, https://github.com/kpihx-labs/uncanny
|
|
23
|
+
Project-URL: Upstream, https://github.com/dunkinfrunkin/uncanny
|
|
24
|
+
Project-URL: Issues, https://github.com/kpihx-labs/uncanny/issues
|
|
25
|
+
Provides-Extra: dev
|
|
26
|
+
Provides-Extra: ml
|
|
27
|
+
Description-Content-Type: text/markdown
|
|
28
|
+
|
|
29
|
+
# Uncanny
|
|
30
|
+
|
|
31
|
+
## KpihX-Labs distribution
|
|
32
|
+
|
|
33
|
+
This MIT-licensed adaptation retains Frank Chan's upstream implementation and history.
|
|
34
|
+
Upstream: https://github.com/dunkinfrunkin/uncanny.
|
|
35
|
+
The PyPI distribution is `k-uncanny`; the executable and Python import remain `uncanny`.
|
|
36
|
+
Install with `uv tool install k-uncanny`, or use `make uv-link` from this checkout.
|
|
37
|
+
Run `uncanny scan document.txt --format json` for structured output on stdout.
|
|
38
|
+
Use `make check`, `make uv-build`, `make uv-publish`, and `make push`.
|
|
39
|
+
`make puhsb` is an alias for `make push`, which pushes the current branch to both
|
|
40
|
+
`github` and `gitlab`, never to `upstream`.
|
|
41
|
+
This detector produces heuristic scores, not calibrated authorship probabilities.
|
|
42
|
+
No score proves who wrote a document or whether plagiarism occurred.
|
|
43
|
+
|
|
44
|
+
The remaining documentation describes the upstream tool. Install instructions naming
|
|
45
|
+
the upstream PyPI package do not install this KpihX-Labs distribution.
|
|
46
|
+
|
|
47
|
+
**Detect AI-generated text from your terminal.** Sentence-level scoring, multiple detection strategies, zero API calls. Runs locally, works offline.
|
|
48
|
+
|
|
49
|
+
```
|
|
50
|
+
$ uncanny scan proposal.md
|
|
51
|
+
|
|
52
|
+
Uncanny Analysis — proposal.md
|
|
53
|
+
─────────────────────────────────────────────────
|
|
54
|
+
|
|
55
|
+
Overall Score: 0.73 (likely AI-generated)
|
|
56
|
+
|
|
57
|
+
██████████████████████░░░░░░░░ 73% AI probability
|
|
58
|
+
|
|
59
|
+
Sentence Breakdown:
|
|
60
|
+
0.91 │ The implementation of this feature requires careful
|
|
61
|
+
│ consideration of several architectural patterns.
|
|
62
|
+
0.42 │ I tried like three different approaches before
|
|
63
|
+
│ landing on this one, honestly.
|
|
64
|
+
0.88 │ By leveraging the existing infrastructure, we can
|
|
65
|
+
│ ensure seamless integration with minimal overhead.
|
|
66
|
+
|
|
67
|
+
Legend: ■ 0.8+ AI ■ 0.5-0.8 mixed ■ <0.5 human
|
|
68
|
+
```
|
|
69
|
+
|
|
70
|
+
## Why Uncanny?
|
|
71
|
+
|
|
72
|
+
Every other AI detector is a SaaS that sends your text to someone else's server. Uncanny runs **entirely on your machine**. No API keys, no subscriptions, no data leaving your network.
|
|
73
|
+
|
|
74
|
+
| Tool | Open Source | Offline | Sentence-level | Multi-method | CLI |
|
|
75
|
+
|------|:----------:|:-------:|:--------------:|:------------:|:---:|
|
|
76
|
+
| GPTZero | | | x | x | |
|
|
77
|
+
| Originality.ai | | | x | x | |
|
|
78
|
+
| Copyleaks | | | x | x | |
|
|
79
|
+
| Sapling | | | x | | |
|
|
80
|
+
| Zippy | x | x | | | x |
|
|
81
|
+
| **Uncanny** | **x** | **x** | **x** | **x** | **x** |
|
|
82
|
+
|
|
83
|
+
## Benchmarks
|
|
84
|
+
|
|
85
|
+
Tested against 36 labeled samples (18 AI-generated, 18 human-written) across diverse genres: academic papers, Slack messages, code reviews, personal essays, Reddit posts, cover letters, tweets, and more.
|
|
86
|
+
|
|
87
|
+
| Metric | Uncanny | Zippy | Random Baseline |
|
|
88
|
+
|--------|:-------:|:-----:|:---------------:|
|
|
89
|
+
| **Overall Accuracy** | **91.7%** | 80.6% | 50.0% |
|
|
90
|
+
| AI Detected (true positive) | 88.9% | 100% | 100% |
|
|
91
|
+
| Human Cleared (true negative) | **94.4%** | 61.1% | 0% |
|
|
92
|
+
| **False Accusation Rate** | **5.6%** | 38.9% | 100% |
|
|
93
|
+
| Speed (per sample) | 8ms | 1ms | 0ms |
|
|
94
|
+
|
|
95
|
+
**Uncanny beats Zippy by 11 points on accuracy.** Zippy catches everything but wrongly flags 39% of human text as AI — that means 2 out of 5 human writers get falsely accused. Uncanny's false positive rate is 7x lower.
|
|
96
|
+
|
|
97
|
+
> Run the benchmarks yourself: `uncanny bench --compare`
|
|
98
|
+
|
|
99
|
+
### How we test
|
|
100
|
+
|
|
101
|
+
The corpus includes adversarial cases:
|
|
102
|
+
|
|
103
|
+
- **Casual AI** — LLM output prompted to sound informal ("Hey! So basically...")
|
|
104
|
+
- **Formal humans** — Academic writing, technical docs, polished prose
|
|
105
|
+
- **Short text** — 2-3 sentence samples where signal is weakest
|
|
106
|
+
- **Edge cases** — LinkedIn posts, PR descriptions, commit messages, debug notes
|
|
107
|
+
|
|
108
|
+
Accuracy is enforced in CI. Every PR must pass `test_bench.py` which gates on:
|
|
109
|
+
- Overall accuracy >= 70%
|
|
110
|
+
- False positive rate < 20%
|
|
111
|
+
- Score separation > 0.10
|
|
112
|
+
|
|
113
|
+
## Install
|
|
114
|
+
|
|
115
|
+
```bash
|
|
116
|
+
pip install uncanny
|
|
117
|
+
```
|
|
118
|
+
|
|
119
|
+
Or with Homebrew:
|
|
120
|
+
|
|
121
|
+
```bash
|
|
122
|
+
brew tap dunkinfrunkin/tap
|
|
123
|
+
brew install uncanny
|
|
124
|
+
```
|
|
125
|
+
|
|
126
|
+
For ML-based analyzers (perplexity):
|
|
127
|
+
|
|
128
|
+
```bash
|
|
129
|
+
pip install "uncanny[ml]"
|
|
130
|
+
```
|
|
131
|
+
|
|
132
|
+
## Usage
|
|
133
|
+
|
|
134
|
+
```bash
|
|
135
|
+
# Scan a file
|
|
136
|
+
uncanny scan report.md
|
|
137
|
+
|
|
138
|
+
# Scan from stdin
|
|
139
|
+
cat essay.txt | uncanny scan
|
|
140
|
+
|
|
141
|
+
# Scan a string directly
|
|
142
|
+
uncanny scan --text "The implementation of this feature..."
|
|
143
|
+
|
|
144
|
+
# Output formats
|
|
145
|
+
uncanny scan report.md --format json # structured JSON
|
|
146
|
+
uncanny scan report.md --format summary # just the score
|
|
147
|
+
|
|
148
|
+
# Speed vs depth
|
|
149
|
+
uncanny scan report.md --fast # compression only (~instant)
|
|
150
|
+
uncanny scan report.md --deep # all analyzers (requires uncanny[ml])
|
|
151
|
+
uncanny scan report.md --analyzers compression,burstiness
|
|
152
|
+
|
|
153
|
+
# CI integration — exit code 1 if score exceeds threshold
|
|
154
|
+
uncanny scan report.md --threshold 0.7
|
|
155
|
+
|
|
156
|
+
# Benchmarks
|
|
157
|
+
uncanny bench # run accuracy benchmarks
|
|
158
|
+
uncanny bench --compare # head-to-head vs competitors
|
|
159
|
+
uncanny bench --compare --json # machine-readable output
|
|
160
|
+
```
|
|
161
|
+
|
|
162
|
+
### JSON Output
|
|
163
|
+
|
|
164
|
+
```json
|
|
165
|
+
{
|
|
166
|
+
"source": "proposal.md",
|
|
167
|
+
"score": 0.73,
|
|
168
|
+
"label": "likely_ai",
|
|
169
|
+
"analyzers": {
|
|
170
|
+
"compression": { "score": 0.81, "method": "lzma+zlib" },
|
|
171
|
+
"burstiness": { "score": 0.71, "variance": 0.003 }
|
|
172
|
+
},
|
|
173
|
+
"sentences": [
|
|
174
|
+
{
|
|
175
|
+
"text": "The implementation of this feature requires...",
|
|
176
|
+
"score": 0.91,
|
|
177
|
+
"offset": { "start": 0, "end": 89 }
|
|
178
|
+
}
|
|
179
|
+
]
|
|
180
|
+
}
|
|
181
|
+
```
|
|
182
|
+
|
|
183
|
+
## How It Works
|
|
184
|
+
|
|
185
|
+
Uncanny combines multiple detection strategies through an **ensemble scorer** with dynamic weighting:
|
|
186
|
+
|
|
187
|
+
### Compression Analysis (primary signal)
|
|
188
|
+
AI text is statistically predictable — it compresses better than human text. Uncanny cross-compresses your text against both an AI-generated prelude and a human-written prelude using LZMA and zlib. If your text shares compression patterns with the AI prelude, it scores higher. This is the fastest method (no model needed) and the most reliable single signal.
|
|
189
|
+
|
|
190
|
+
### Burstiness Analysis (tiebreaker)
|
|
191
|
+
Humans write with natural variation — short sentences mixed with long ones, simple ideas next to complex ones. AI text is uniformly smooth. Uncanny measures the variance in compression ratio and sentence length across a document. Low variance = AI-like uniformity. This signal gets dynamically weighted — it matters most when compression is ambiguous.
|
|
192
|
+
|
|
193
|
+
### Perplexity Analysis (optional, requires `uncanny[ml]`)
|
|
194
|
+
Measures how "surprised" GPT-2 is by each sentence. AI-generated text has low perplexity (the model predicted it easily). Human text has high perplexity (surprising word choices). Requires downloading a ~500MB model.
|
|
195
|
+
|
|
196
|
+
### Ensemble Scoring
|
|
197
|
+
Analyzers are combined with **confidence-aware dynamic weighting**. When compression is decisive (far from 0.5), it dominates. When compression is ambiguous, burstiness gets more influence. This avoids the classic problem of a weak signal overriding a strong one.
|
|
198
|
+
|
|
199
|
+
## Design Principles
|
|
200
|
+
|
|
201
|
+
1. **Offline-first** — No API calls. Everything runs locally. Models downloaded once.
|
|
202
|
+
2. **Fast by default** — Default mode uses compression + burstiness (~8ms/sample). No model download needed.
|
|
203
|
+
3. **Scores, not verdicts** — Uncanny reports probabilities, never binary "AI or human" labels. The tool informs, it doesn't judge.
|
|
204
|
+
4. **Sentence-level granularity** — Not just "this doc is AI" — shows exactly which sentences triggered detection.
|
|
205
|
+
5. **CI-friendly** — `--threshold` flag with exit codes. JSON output for parsing. Benchmarks in CI.
|
|
206
|
+
|
|
207
|
+
## Limitations
|
|
208
|
+
|
|
209
|
+
AI detection is an imperfect science. Be honest about it:
|
|
210
|
+
|
|
211
|
+
- **No detector is 100% accurate.** False positives and false negatives happen.
|
|
212
|
+
- **Short text is unreliable.** Confidence drops below ~250 words.
|
|
213
|
+
- **Edited AI text is harder to detect.** A human editing AI output blurs the signal.
|
|
214
|
+
- **Newer models are harder to detect.** As LLMs improve, statistical signals narrow.
|
|
215
|
+
- **Technical writing may false-positive.** Formal, structured text can look AI-like.
|
|
216
|
+
|
|
217
|
+
## Contributing
|
|
218
|
+
|
|
219
|
+
See [CONTRIBUTING.md](CONTRIBUTING.md). PRs welcome — especially new analyzers, corpus samples, and accuracy improvements.
|
|
220
|
+
|
|
221
|
+
## License
|
|
222
|
+
|
|
223
|
+
MIT
|
|
@@ -0,0 +1,195 @@
|
|
|
1
|
+
# Uncanny
|
|
2
|
+
|
|
3
|
+
## KpihX-Labs distribution
|
|
4
|
+
|
|
5
|
+
This MIT-licensed adaptation retains Frank Chan's upstream implementation and history.
|
|
6
|
+
Upstream: https://github.com/dunkinfrunkin/uncanny.
|
|
7
|
+
The PyPI distribution is `k-uncanny`; the executable and Python import remain `uncanny`.
|
|
8
|
+
Install with `uv tool install k-uncanny`, or use `make uv-link` from this checkout.
|
|
9
|
+
Run `uncanny scan document.txt --format json` for structured output on stdout.
|
|
10
|
+
Use `make check`, `make uv-build`, `make uv-publish`, and `make push`.
|
|
11
|
+
`make puhsb` is an alias for `make push`, which pushes the current branch to both
|
|
12
|
+
`github` and `gitlab`, never to `upstream`.
|
|
13
|
+
This detector produces heuristic scores, not calibrated authorship probabilities.
|
|
14
|
+
No score proves who wrote a document or whether plagiarism occurred.
|
|
15
|
+
|
|
16
|
+
The remaining documentation describes the upstream tool. Install instructions naming
|
|
17
|
+
the upstream PyPI package do not install this KpihX-Labs distribution.
|
|
18
|
+
|
|
19
|
+
**Detect AI-generated text from your terminal.** Sentence-level scoring, multiple detection strategies, zero API calls. Runs locally, works offline.
|
|
20
|
+
|
|
21
|
+
```
|
|
22
|
+
$ uncanny scan proposal.md
|
|
23
|
+
|
|
24
|
+
Uncanny Analysis — proposal.md
|
|
25
|
+
─────────────────────────────────────────────────
|
|
26
|
+
|
|
27
|
+
Overall Score: 0.73 (likely AI-generated)
|
|
28
|
+
|
|
29
|
+
██████████████████████░░░░░░░░ 73% AI probability
|
|
30
|
+
|
|
31
|
+
Sentence Breakdown:
|
|
32
|
+
0.91 │ The implementation of this feature requires careful
|
|
33
|
+
│ consideration of several architectural patterns.
|
|
34
|
+
0.42 │ I tried like three different approaches before
|
|
35
|
+
│ landing on this one, honestly.
|
|
36
|
+
0.88 │ By leveraging the existing infrastructure, we can
|
|
37
|
+
│ ensure seamless integration with minimal overhead.
|
|
38
|
+
|
|
39
|
+
Legend: ■ 0.8+ AI ■ 0.5-0.8 mixed ■ <0.5 human
|
|
40
|
+
```
|
|
41
|
+
|
|
42
|
+
## Why Uncanny?
|
|
43
|
+
|
|
44
|
+
Every other AI detector is a SaaS that sends your text to someone else's server. Uncanny runs **entirely on your machine**. No API keys, no subscriptions, no data leaving your network.
|
|
45
|
+
|
|
46
|
+
| Tool | Open Source | Offline | Sentence-level | Multi-method | CLI |
|
|
47
|
+
|------|:----------:|:-------:|:--------------:|:------------:|:---:|
|
|
48
|
+
| GPTZero | | | x | x | |
|
|
49
|
+
| Originality.ai | | | x | x | |
|
|
50
|
+
| Copyleaks | | | x | x | |
|
|
51
|
+
| Sapling | | | x | | |
|
|
52
|
+
| Zippy | x | x | | | x |
|
|
53
|
+
| **Uncanny** | **x** | **x** | **x** | **x** | **x** |
|
|
54
|
+
|
|
55
|
+
## Benchmarks
|
|
56
|
+
|
|
57
|
+
Tested against 36 labeled samples (18 AI-generated, 18 human-written) across diverse genres: academic papers, Slack messages, code reviews, personal essays, Reddit posts, cover letters, tweets, and more.
|
|
58
|
+
|
|
59
|
+
| Metric | Uncanny | Zippy | Random Baseline |
|
|
60
|
+
|--------|:-------:|:-----:|:---------------:|
|
|
61
|
+
| **Overall Accuracy** | **91.7%** | 80.6% | 50.0% |
|
|
62
|
+
| AI Detected (true positive) | 88.9% | 100% | 100% |
|
|
63
|
+
| Human Cleared (true negative) | **94.4%** | 61.1% | 0% |
|
|
64
|
+
| **False Accusation Rate** | **5.6%** | 38.9% | 100% |
|
|
65
|
+
| Speed (per sample) | 8ms | 1ms | 0ms |
|
|
66
|
+
|
|
67
|
+
**Uncanny beats Zippy by 11 points on accuracy.** Zippy catches everything but wrongly flags 39% of human text as AI — that means 2 out of 5 human writers get falsely accused. Uncanny's false positive rate is 7x lower.
|
|
68
|
+
|
|
69
|
+
> Run the benchmarks yourself: `uncanny bench --compare`
|
|
70
|
+
|
|
71
|
+
### How we test
|
|
72
|
+
|
|
73
|
+
The corpus includes adversarial cases:
|
|
74
|
+
|
|
75
|
+
- **Casual AI** — LLM output prompted to sound informal ("Hey! So basically...")
|
|
76
|
+
- **Formal humans** — Academic writing, technical docs, polished prose
|
|
77
|
+
- **Short text** — 2-3 sentence samples where signal is weakest
|
|
78
|
+
- **Edge cases** — LinkedIn posts, PR descriptions, commit messages, debug notes
|
|
79
|
+
|
|
80
|
+
Accuracy is enforced in CI. Every PR must pass `test_bench.py` which gates on:
|
|
81
|
+
- Overall accuracy >= 70%
|
|
82
|
+
- False positive rate < 20%
|
|
83
|
+
- Score separation > 0.10
|
|
84
|
+
|
|
85
|
+
## Install
|
|
86
|
+
|
|
87
|
+
```bash
|
|
88
|
+
pip install uncanny
|
|
89
|
+
```
|
|
90
|
+
|
|
91
|
+
Or with Homebrew:
|
|
92
|
+
|
|
93
|
+
```bash
|
|
94
|
+
brew tap dunkinfrunkin/tap
|
|
95
|
+
brew install uncanny
|
|
96
|
+
```
|
|
97
|
+
|
|
98
|
+
For ML-based analyzers (perplexity):
|
|
99
|
+
|
|
100
|
+
```bash
|
|
101
|
+
pip install "uncanny[ml]"
|
|
102
|
+
```
|
|
103
|
+
|
|
104
|
+
## Usage
|
|
105
|
+
|
|
106
|
+
```bash
|
|
107
|
+
# Scan a file
|
|
108
|
+
uncanny scan report.md
|
|
109
|
+
|
|
110
|
+
# Scan from stdin
|
|
111
|
+
cat essay.txt | uncanny scan
|
|
112
|
+
|
|
113
|
+
# Scan a string directly
|
|
114
|
+
uncanny scan --text "The implementation of this feature..."
|
|
115
|
+
|
|
116
|
+
# Output formats
|
|
117
|
+
uncanny scan report.md --format json # structured JSON
|
|
118
|
+
uncanny scan report.md --format summary # just the score
|
|
119
|
+
|
|
120
|
+
# Speed vs depth
|
|
121
|
+
uncanny scan report.md --fast # compression only (~instant)
|
|
122
|
+
uncanny scan report.md --deep # all analyzers (requires uncanny[ml])
|
|
123
|
+
uncanny scan report.md --analyzers compression,burstiness
|
|
124
|
+
|
|
125
|
+
# CI integration — exit code 1 if score exceeds threshold
|
|
126
|
+
uncanny scan report.md --threshold 0.7
|
|
127
|
+
|
|
128
|
+
# Benchmarks
|
|
129
|
+
uncanny bench # run accuracy benchmarks
|
|
130
|
+
uncanny bench --compare # head-to-head vs competitors
|
|
131
|
+
uncanny bench --compare --json # machine-readable output
|
|
132
|
+
```
|
|
133
|
+
|
|
134
|
+
### JSON Output
|
|
135
|
+
|
|
136
|
+
```json
|
|
137
|
+
{
|
|
138
|
+
"source": "proposal.md",
|
|
139
|
+
"score": 0.73,
|
|
140
|
+
"label": "likely_ai",
|
|
141
|
+
"analyzers": {
|
|
142
|
+
"compression": { "score": 0.81, "method": "lzma+zlib" },
|
|
143
|
+
"burstiness": { "score": 0.71, "variance": 0.003 }
|
|
144
|
+
},
|
|
145
|
+
"sentences": [
|
|
146
|
+
{
|
|
147
|
+
"text": "The implementation of this feature requires...",
|
|
148
|
+
"score": 0.91,
|
|
149
|
+
"offset": { "start": 0, "end": 89 }
|
|
150
|
+
}
|
|
151
|
+
]
|
|
152
|
+
}
|
|
153
|
+
```
|
|
154
|
+
|
|
155
|
+
## How It Works
|
|
156
|
+
|
|
157
|
+
Uncanny combines multiple detection strategies through an **ensemble scorer** with dynamic weighting:
|
|
158
|
+
|
|
159
|
+
### Compression Analysis (primary signal)
|
|
160
|
+
AI text is statistically predictable — it compresses better than human text. Uncanny cross-compresses your text against both an AI-generated prelude and a human-written prelude using LZMA and zlib. If your text shares compression patterns with the AI prelude, it scores higher. This is the fastest method (no model needed) and the most reliable single signal.
|
|
161
|
+
|
|
162
|
+
### Burstiness Analysis (tiebreaker)
|
|
163
|
+
Humans write with natural variation — short sentences mixed with long ones, simple ideas next to complex ones. AI text is uniformly smooth. Uncanny measures the variance in compression ratio and sentence length across a document. Low variance = AI-like uniformity. This signal gets dynamically weighted — it matters most when compression is ambiguous.
|
|
164
|
+
|
|
165
|
+
### Perplexity Analysis (optional, requires `uncanny[ml]`)
|
|
166
|
+
Measures how "surprised" GPT-2 is by each sentence. AI-generated text has low perplexity (the model predicted it easily). Human text has high perplexity (surprising word choices). Requires downloading a ~500MB model.
|
|
167
|
+
|
|
168
|
+
### Ensemble Scoring
|
|
169
|
+
Analyzers are combined with **confidence-aware dynamic weighting**. When compression is decisive (far from 0.5), it dominates. When compression is ambiguous, burstiness gets more influence. This avoids the classic problem of a weak signal overriding a strong one.
|
|
170
|
+
|
|
171
|
+
## Design Principles
|
|
172
|
+
|
|
173
|
+
1. **Offline-first** — No API calls. Everything runs locally. Models downloaded once.
|
|
174
|
+
2. **Fast by default** — Default mode uses compression + burstiness (~8ms/sample). No model download needed.
|
|
175
|
+
3. **Scores, not verdicts** — Uncanny reports probabilities, never binary "AI or human" labels. The tool informs, it doesn't judge.
|
|
176
|
+
4. **Sentence-level granularity** — Not just "this doc is AI" — shows exactly which sentences triggered detection.
|
|
177
|
+
5. **CI-friendly** — `--threshold` flag with exit codes. JSON output for parsing. Benchmarks in CI.
|
|
178
|
+
|
|
179
|
+
## Limitations
|
|
180
|
+
|
|
181
|
+
AI detection is an imperfect science. Be honest about it:
|
|
182
|
+
|
|
183
|
+
- **No detector is 100% accurate.** False positives and false negatives happen.
|
|
184
|
+
- **Short text is unreliable.** Confidence drops below ~250 words.
|
|
185
|
+
- **Edited AI text is harder to detect.** A human editing AI output blurs the signal.
|
|
186
|
+
- **Newer models are harder to detect.** As LLMs improve, statistical signals narrow.
|
|
187
|
+
- **Technical writing may false-positive.** Formal, structured text can look AI-like.
|
|
188
|
+
|
|
189
|
+
## Contributing
|
|
190
|
+
|
|
191
|
+
See [CONTRIBUTING.md](CONTRIBUTING.md). PRs welcome — especially new analyzers, corpus samples, and accuracy improvements.
|
|
192
|
+
|
|
193
|
+
## License
|
|
194
|
+
|
|
195
|
+
MIT
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["uv_build>=0.10.3,<0.11.0"]
|
|
3
|
+
build-backend = "uv_build"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "k-uncanny"
|
|
7
|
+
version = "0.1.1"
|
|
8
|
+
description = "CLI tool that detects AI-generated text with sentence-level scoring and multiple detection strategies"
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
license = "MIT"
|
|
11
|
+
requires-python = ">=3.10"
|
|
12
|
+
authors = [{ name = "Frank Chan" }]
|
|
13
|
+
urls = { Homepage = "https://github.com/kpihx-labs/uncanny", Repository = "https://github.com/kpihx-labs/uncanny", Upstream = "https://github.com/dunkinfrunkin/uncanny", Issues = "https://github.com/kpihx-labs/uncanny/issues" }
|
|
14
|
+
keywords = ["ai-detection", "gpt", "llm", "text-analysis", "cli"]
|
|
15
|
+
classifiers = [
|
|
16
|
+
"Development Status :: 3 - Alpha",
|
|
17
|
+
"Environment :: Console",
|
|
18
|
+
"Intended Audience :: Developers",
|
|
19
|
+
"License :: OSI Approved :: MIT License",
|
|
20
|
+
"Programming Language :: Python :: 3",
|
|
21
|
+
"Topic :: Text Processing :: Linguistic",
|
|
22
|
+
]
|
|
23
|
+
dependencies = [
|
|
24
|
+
"typer>=0.9",
|
|
25
|
+
"rich>=13.0",
|
|
26
|
+
]
|
|
27
|
+
|
|
28
|
+
[project.optional-dependencies]
|
|
29
|
+
ml = [
|
|
30
|
+
"torch>=2.0",
|
|
31
|
+
"transformers>=4.30",
|
|
32
|
+
]
|
|
33
|
+
dev = [
|
|
34
|
+
"pytest>=7.0",
|
|
35
|
+
"pytest-cov>=4.0",
|
|
36
|
+
]
|
|
37
|
+
|
|
38
|
+
[project.scripts]
|
|
39
|
+
uncanny = "uncanny.cli:app"
|
|
40
|
+
|
|
41
|
+
[tool.uv.build-backend]
|
|
42
|
+
module-name = "uncanny"
|
|
43
|
+
|
|
44
|
+
[tool.pytest.ini_options]
|
|
45
|
+
testpaths = ["tests"]
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
from uncanny.analyzers.base import Analyzer
|
|
2
|
+
from uncanny.analyzers.compression import CompressionAnalyzer
|
|
3
|
+
from uncanny.analyzers.burstiness import BurstinessAnalyzer
|
|
4
|
+
|
|
5
|
+
__all__ = ["Analyzer", "CompressionAnalyzer", "BurstinessAnalyzer"]
|
|
6
|
+
|
|
7
|
+
# Perplexity analyzer requires optional ML dependencies
|
|
8
|
+
try:
|
|
9
|
+
from uncanny.analyzers.perplexity import PerplexityAnalyzer # noqa: F401
|
|
10
|
+
|
|
11
|
+
__all__.append("PerplexityAnalyzer")
|
|
12
|
+
except ImportError:
|
|
13
|
+
pass
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from abc import ABC, abstractmethod
|
|
4
|
+
|
|
5
|
+
from uncanny.types import AnalyzerResult
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class Analyzer(ABC):
|
|
9
|
+
"""Base class for all detection analyzers."""
|
|
10
|
+
|
|
11
|
+
name: str
|
|
12
|
+
|
|
13
|
+
@abstractmethod
|
|
14
|
+
def analyze(self, text: str) -> AnalyzerResult:
|
|
15
|
+
"""Analyze text and return a result with overall + sentence scores."""
|
|
16
|
+
...
|
|
@@ -0,0 +1,121 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Burstiness-based AI detection.
|
|
3
|
+
|
|
4
|
+
Measures variance in sentence complexity across a document.
|
|
5
|
+
Humans write with natural variation (short/long sentences, simple/complex ideas).
|
|
6
|
+
AI text is uniformly smooth.
|
|
7
|
+
|
|
8
|
+
Uses zlib compression ratio per-sentence as a proxy for complexity.
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
import math
|
|
14
|
+
import zlib
|
|
15
|
+
|
|
16
|
+
from uncanny.analyzers.base import Analyzer
|
|
17
|
+
from uncanny.tokenizer import split_sentences
|
|
18
|
+
from uncanny.types import AnalyzerResult, SentenceScore
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def _compress_ratio(data: bytes) -> float:
|
|
22
|
+
"""Compression ratio using zlib."""
|
|
23
|
+
if len(data) == 0:
|
|
24
|
+
return 1.0
|
|
25
|
+
return len(zlib.compress(data, level=6)) / len(data)
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def _variance(values: list[float]) -> float:
|
|
29
|
+
if len(values) < 2:
|
|
30
|
+
return 0.0
|
|
31
|
+
mean = sum(values) / len(values)
|
|
32
|
+
return sum((v - mean) ** 2 for v in values) / (len(values) - 1)
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def _burstiness_to_score(variance: float, length_cv: float) -> float:
|
|
36
|
+
"""Convert burstiness metrics to AI probability score.
|
|
37
|
+
|
|
38
|
+
Low variance + low length CV = uniform = AI-like = high score.
|
|
39
|
+
High variance + high length CV = bursty = human-like = low score.
|
|
40
|
+
|
|
41
|
+
Calibrated from corpus:
|
|
42
|
+
AI: ratio_var mean=0.020, length_cv mean=0.41
|
|
43
|
+
Human: ratio_var mean=0.041, length_cv mean=0.62
|
|
44
|
+
"""
|
|
45
|
+
# Compression ratio variance signal
|
|
46
|
+
# AI: 0.001-0.070, mean 0.020. Human: 0.003-0.079, mean 0.041.
|
|
47
|
+
# Use sigmoid centered around 0.030 (midpoint)
|
|
48
|
+
rv_centered = (variance - 0.030) * 60.0
|
|
49
|
+
ratio_signal = 1.0 / (1.0 + math.exp(rv_centered))
|
|
50
|
+
|
|
51
|
+
# Sentence length CV signal
|
|
52
|
+
# AI: 0.17-1.12, mean 0.41. Human: 0.22-0.89, mean 0.62.
|
|
53
|
+
# Use sigmoid centered around 0.50 (midpoint)
|
|
54
|
+
lcv_centered = (length_cv - 0.50) * 4.0
|
|
55
|
+
length_signal = 1.0 / (1.0 + math.exp(lcv_centered))
|
|
56
|
+
|
|
57
|
+
return 0.55 * ratio_signal + 0.45 * length_signal
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
class BurstinessAnalyzer(Analyzer):
|
|
61
|
+
"""Detects AI text by measuring uniformity of sentence complexity."""
|
|
62
|
+
|
|
63
|
+
name = "burstiness"
|
|
64
|
+
|
|
65
|
+
def analyze(self, text: str) -> AnalyzerResult:
|
|
66
|
+
sentences = split_sentences(text)
|
|
67
|
+
|
|
68
|
+
if len(sentences) < 3:
|
|
69
|
+
return AnalyzerResult(
|
|
70
|
+
name=self.name,
|
|
71
|
+
score=0.5,
|
|
72
|
+
sentence_scores=[
|
|
73
|
+
SentenceScore(s, 0.5, start, end) for s, start, end in sentences
|
|
74
|
+
],
|
|
75
|
+
metadata={"variance": 0.0, "note": "too few sentences"},
|
|
76
|
+
)
|
|
77
|
+
|
|
78
|
+
# Compression ratio per sentence as complexity proxy
|
|
79
|
+
ratios: list[float] = []
|
|
80
|
+
for sent_text, _, _ in sentences:
|
|
81
|
+
if len(sent_text.strip()) < 10:
|
|
82
|
+
continue
|
|
83
|
+
ratios.append(_compress_ratio(sent_text.encode("utf-8")))
|
|
84
|
+
|
|
85
|
+
if len(ratios) < 3:
|
|
86
|
+
return AnalyzerResult(
|
|
87
|
+
name=self.name,
|
|
88
|
+
score=0.5,
|
|
89
|
+
sentence_scores=[
|
|
90
|
+
SentenceScore(s, 0.5, start, end) for s, start, end in sentences
|
|
91
|
+
],
|
|
92
|
+
metadata={"variance": 0.0, "note": "too few analyzable sentences"},
|
|
93
|
+
)
|
|
94
|
+
|
|
95
|
+
# Sentence length variance
|
|
96
|
+
lengths = [len(s.split()) for s, _, _ in sentences]
|
|
97
|
+
mean_length = sum(lengths) / len(lengths)
|
|
98
|
+
length_cv = math.sqrt(_variance(lengths)) / max(mean_length, 1)
|
|
99
|
+
|
|
100
|
+
ratio_var = _variance(ratios)
|
|
101
|
+
overall_score = _burstiness_to_score(ratio_var, length_cv)
|
|
102
|
+
|
|
103
|
+
# Per-sentence: deviation from mean indicates human-like variety
|
|
104
|
+
mean_ratio = sum(ratios) / len(ratios)
|
|
105
|
+
sentence_scores: list[SentenceScore] = []
|
|
106
|
+
ratio_idx = 0
|
|
107
|
+
for sent_text, start, end in sentences:
|
|
108
|
+
if len(sent_text.strip()) < 10:
|
|
109
|
+
sentence_scores.append(SentenceScore(sent_text, 0.5, start, end))
|
|
110
|
+
continue
|
|
111
|
+
deviation = abs(ratios[ratio_idx] - mean_ratio)
|
|
112
|
+
sent_score = max(0.0, min(1.0, 1.0 - deviation * 15))
|
|
113
|
+
sentence_scores.append(SentenceScore(sent_text, sent_score, start, end))
|
|
114
|
+
ratio_idx += 1
|
|
115
|
+
|
|
116
|
+
return AnalyzerResult(
|
|
117
|
+
name=self.name,
|
|
118
|
+
score=overall_score,
|
|
119
|
+
sentence_scores=sentence_scores,
|
|
120
|
+
metadata={"variance": round(ratio_var, 6), "length_cv": round(length_cv, 3)},
|
|
121
|
+
)
|