aidetect 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- aidetect-0.1.0/.github/workflows/publish.yml +41 -0
- aidetect-0.1.0/.gitignore +22 -0
- aidetect-0.1.0/LICENSE +21 -0
- aidetect-0.1.0/PKG-INFO +236 -0
- aidetect-0.1.0/PRODUCT.md +65 -0
- aidetect-0.1.0/README.md +216 -0
- aidetect-0.1.0/aidetect-cli/SKILL.md +99 -0
- aidetect-0.1.0/pyproject.toml +41 -0
- aidetect-0.1.0/src/aidetect/__init__.py +1 -0
- aidetect-0.1.0/src/aidetect/binoculars.py +243 -0
- aidetect-0.1.0/src/aidetect/calibrate.py +123 -0
- aidetect-0.1.0/src/aidetect/cli.py +46 -0
- aidetect-0.1.0/src/aidetect/count.py +118 -0
- aidetect-0.1.0/src/aidetect/detect.py +121 -0
- aidetect-0.1.0/src/aidetect/extract.py +40 -0
- aidetect-0.1.0/src/aidetect/paths.py +32 -0
- aidetect-0.1.0/src/aidetect/text.py +64 -0
- aidetect-0.1.0/src/aidetect/thresholds/threshold-big.json +12 -0
- aidetect-0.1.0/src/aidetect/thresholds/threshold-gemma-mlx.json +13 -0
- aidetect-0.1.0/src/aidetect/thresholds/threshold-small.json +12 -0
- aidetect-0.1.0/tests/test_binoculars.py +67 -0
- aidetect-0.1.0/tests/test_count.py +126 -0
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
# Publishes to PyPI when a GitHub release is published.
|
|
2
|
+
#
|
|
3
|
+
# No token anywhere: PyPI trusts this exact workflow in this exact repo through
|
|
4
|
+
# OpenID Connect, which is why `id-token: write` and the `pypi` environment below
|
|
5
|
+
# are not optional โ they are the identity PyPI checks against its publisher config.
|
|
6
|
+
#
|
|
7
|
+
# To ship a version: bump `version` in pyproject.toml, commit, then draft a release
|
|
8
|
+
# on GitHub tagged v<version>. Nothing is built or uploaded from a laptop.
|
|
9
|
+
name: publish
|
|
10
|
+
|
|
11
|
+
on:
|
|
12
|
+
release:
|
|
13
|
+
types: [published]
|
|
14
|
+
|
|
15
|
+
jobs:
|
|
16
|
+
build:
|
|
17
|
+
runs-on: ubuntu-latest
|
|
18
|
+
steps:
|
|
19
|
+
- uses: actions/checkout@v4
|
|
20
|
+
- uses: astral-sh/setup-uv@v5
|
|
21
|
+
- run: uv build
|
|
22
|
+
# The wheel is py3-none-any, so a Linux runner builds the same artifact a Mac
|
|
23
|
+
# would. The mlx-vlm dependency is a marker, not a build input.
|
|
24
|
+
- run: uv run --no-project --with python-docx python tests/test_count.py
|
|
25
|
+
- uses: actions/upload-artifact@v4
|
|
26
|
+
with:
|
|
27
|
+
name: dist
|
|
28
|
+
path: dist/
|
|
29
|
+
|
|
30
|
+
publish:
|
|
31
|
+
needs: build
|
|
32
|
+
runs-on: ubuntu-latest
|
|
33
|
+
environment: pypi
|
|
34
|
+
permissions:
|
|
35
|
+
id-token: write
|
|
36
|
+
steps:
|
|
37
|
+
- uses: actions/download-artifact@v4
|
|
38
|
+
with:
|
|
39
|
+
name: dist
|
|
40
|
+
path: dist/
|
|
41
|
+
- uses: pypa/gh-action-pypi-publish@release/v1
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
# personal drafts โ never publish my own essays
|
|
2
|
+
EE-clean.txt
|
|
3
|
+
*.docx
|
|
4
|
+
*-clean.txt
|
|
5
|
+
|
|
6
|
+
# second-opinion tool is an external repo, not mine to redistribute
|
|
7
|
+
fast-ai-detector/
|
|
8
|
+
|
|
9
|
+
.DS_Store
|
|
10
|
+
|
|
11
|
+
# python
|
|
12
|
+
__pycache__/
|
|
13
|
+
*.pyc
|
|
14
|
+
.venv/
|
|
15
|
+
|
|
16
|
+
# huggingface model cache
|
|
17
|
+
.cache/
|
|
18
|
+
|
|
19
|
+
# build artifacts
|
|
20
|
+
dist/
|
|
21
|
+
*.egg-info/
|
|
22
|
+
uv.lock
|
aidetect-0.1.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Nick Trimandylis
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
aidetect-0.1.0/PKG-INFO
ADDED
|
@@ -0,0 +1,236 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: aidetect
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Local, offline AI-writing detector and IB word counter for your own drafts
|
|
5
|
+
Project-URL: Homepage, https://github.com/nitrimandylis/aidetect
|
|
6
|
+
Project-URL: Source, https://github.com/nitrimandylis/aidetect
|
|
7
|
+
Author: Nikolas Trimandylis
|
|
8
|
+
License-Expression: MIT
|
|
9
|
+
License-File: LICENSE
|
|
10
|
+
Keywords: ai-detection,binoculars,docx,ib,word-count
|
|
11
|
+
Classifier: Environment :: Console
|
|
12
|
+
Classifier: Programming Language :: Python :: 3
|
|
13
|
+
Classifier: Topic :: Text Processing :: Linguistic
|
|
14
|
+
Requires-Python: >=3.10
|
|
15
|
+
Requires-Dist: mlx-vlm; sys_platform == 'darwin' and platform_machine == 'arm64'
|
|
16
|
+
Requires-Dist: python-docx
|
|
17
|
+
Requires-Dist: torch
|
|
18
|
+
Requires-Dist: transformers
|
|
19
|
+
Description-Content-Type: text/markdown
|
|
20
|
+
|
|
21
|
+
```
|
|
22
|
+
โโโโโโ โโโ โโโโโโโ โโโโโโโโโโโโโโโโโโโโโโโโโ โโโโโโโโโโโโโโโโ
|
|
23
|
+
โโโโโโโโโโโ โโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโ
|
|
24
|
+
โโโโโโโโโโโโโโโโโโโโ โโโโโโโโโ โโโ โโโโโโ โโโ โโโ
|
|
25
|
+
โโโโโโโโโโโโโโโโโโโโ โโโโโโโโโ โโโ โโโโโโ โโโ โโโ
|
|
26
|
+
โโโ โโโโโโ โโโโโโโโโโโโโโโโ โโโ โโโโโโโโโโโโโโโโ โโโ
|
|
27
|
+
โโโ โโโโโโ โโโโโโโ โโโโโโโโ โโโ โโโโโโโโ โโโโโโโ โโโ
|
|
28
|
+
```
|
|
29
|
+
<div align="center">
|
|
30
|
+
|
|
31
|
+
### `SCORE YOUR OWN PROSE // BEFORE A TEACHER SCORES IT FOR YOU`
|
|
32
|
+
|
|
33
|
+
*a local, offline AI-writing detector and IB word counter for drafts you actually wrote*
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
-111111?style=flat-square&labelColor=111111)
|
|
41
|
+
|
|
42
|
+
</div>
|
|
43
|
+
|
|
44
|
+
---
|
|
45
|
+
|
|
46
|
+
## ๐ What is this
|
|
47
|
+
|
|
48
|
+
A command-line tool that reads a `.docx` or `.txt` and scores each paragraph
|
|
49
|
+
0โ1 on how AI-generated it reads, using the `desklib/ai-text-detector-v1.01`
|
|
50
|
+
DeBERTa model โ the one sitting at #1 on the RAID benchmark. Everything runs
|
|
51
|
+
on your own machine; after the first model download it never touches the
|
|
52
|
+
network. You point it at your Extended Essay, it tells you which paragraphs
|
|
53
|
+
sound like a language model wrote them.
|
|
54
|
+
|
|
55
|
+
The point isn't to cheat a detector. It's the opposite: I write my own drafts,
|
|
56
|
+
and sometimes my own honest prose still trips these classifiers because that's
|
|
57
|
+
what earnest formal writing looks like to them. This flags those paragraphs so
|
|
58
|
+
I can reword before a teacher runs Turnitin and has an awkward conversation
|
|
59
|
+
with me about it.
|
|
60
|
+
|
|
61
|
+
It is directional, not oracular. A high score means "reword this," not "you're
|
|
62
|
+
caught." It is not, and cannot be, the number Turnitin shows a teacher.
|
|
63
|
+
|
|
64
|
+
```console
|
|
65
|
+
nick@aidetect:~$ aidetect score EE-clean.txt
|
|
66
|
+
P 1 0.08 [##------------------]
|
|
67
|
+
P 2 0.71 [##############------] <-- AI-ish
|
|
68
|
+
average AI score: 0.34 | 1/6 paragraphs flagged
|
|
69
|
+
reminder: directional only, not a Turnitin score.
|
|
70
|
+
```
|
|
71
|
+
|
|
72
|
+
## ๐ง The detection engine
|
|
73
|
+
|
|
74
|
+
| | feature | what it actually does |
|
|
75
|
+
|---|---|---|
|
|
76
|
+
| 01 | **per-paragraph scoring** | what it actually catches โ splits your draft and scores each paragraph, so you fix the two bad ones instead of rewriting everything |
|
|
77
|
+
| 02 | **docx + txt input** | reads Word files straight (paragraphs, no headings) or plain text split on blank lines |
|
|
78
|
+
| 03 | **prose extractor** | `aidetect extract` strips headings, bullets, footnotes and your own note-scaffolding first, so the score is about writing, not structure |
|
|
79
|
+
| 04 | **offline after setup** | first run pulls ~1.5GB of model, every run after is airgapped โ your essay never leaves the laptop |
|
|
80
|
+
| 05 | **second opinion** | cross-check against the lighter [Ejhfast/fast-ai-detector] when one model's paranoia isn't enough |
|
|
81
|
+
| 06 | **Binoculars (Gemma 4)** | a training-free perplexity-ratio detector โ near chance with small Qwen pairs, but 96% on the labelled set once swapped to a Gemma 4 pair; see below |
|
|
82
|
+
| 07 | **IB word count** | `aidetect count` (`--json` for scripts and agents) counts what the IB counts โ no headings, quotes, tables, footnotes, citations or bibliography โ and splits the total by section, so an over-long draft tells you *where* |
|
|
83
|
+
|
|
84
|
+
## ๐ Run it
|
|
85
|
+
|
|
86
|
+
```bash
|
|
87
|
+
uv tool install aidetect # or: pipx install aidetect
|
|
88
|
+
```
|
|
89
|
+
|
|
90
|
+
```bash
|
|
91
|
+
aidetect # list the five subcommands
|
|
92
|
+
aidetect count "draft.docx" --limit 4000 # IB word count, by section
|
|
93
|
+
aidetect count "draft.docx" --json # same, as one JSON object
|
|
94
|
+
aidetect score "draft.docx" # score a whole draft
|
|
95
|
+
aidetect score --text "one sentence" # score a single string
|
|
96
|
+
aidetect bino "draft.docx" --mlx --pair gemma
|
|
97
|
+
```
|
|
98
|
+
|
|
99
|
+
`count` and `extract` are instant and need no model. `score` and `bino` need a
|
|
100
|
+
machine that can hold a transformer: built and tested on an 18GB Apple Silicon
|
|
101
|
+
Mac, MPS-accelerated. Their first run downloads the model and will sit there for
|
|
102
|
+
a minute โ that's normal, not a hang. Every run after is fast and offline.
|
|
103
|
+
|
|
104
|
+
On Apple Silicon the Gemma 4 MLX pair installs automatically. Elsewhere it is
|
|
105
|
+
skipped and the Qwen pairs still work.
|
|
106
|
+
|
|
107
|
+
### `--json`
|
|
108
|
+
|
|
109
|
+
`count` takes `--json` and prints exactly one object on stdout, nothing else:
|
|
110
|
+
|
|
111
|
+
```json
|
|
112
|
+
{"sections": [{"title": "Introduction", "words": 812}], "total": 3940, "limit": 4000, "over": -60}
|
|
113
|
+
```
|
|
114
|
+
|
|
115
|
+
Every key is always present. `limit` and `over` are `null` when no `--limit` was
|
|
116
|
+
given โ that means "does not apply", not "could not be read". A draft with no
|
|
117
|
+
prose is an empty `sections` list and exit 0. Errors go to stderr with a non-zero
|
|
118
|
+
exit, so consumers branch on the exit code rather than parsing error text.
|
|
119
|
+
|
|
120
|
+
## ๐ฉ Under the hood
|
|
121
|
+
|
|
122
|
+
```mermaid
|
|
123
|
+
flowchart LR
|
|
124
|
+
A[.docx / .txt] --> B[aidetect extract<br/>strip non-prose]
|
|
125
|
+
B --> C[read_paragraphs<br/>>= 25 words]
|
|
126
|
+
C --> D[desklib DeBERTa<br/>mean-pool + sigmoid]
|
|
127
|
+
D --> E[per-paragraph<br/>0-1 score + flags]
|
|
128
|
+
```
|
|
129
|
+
|
|
130
|
+
| file | job |
|
|
131
|
+
|---|---|
|
|
132
|
+
| `src/aidetect/cli.py` | the `aidetect` entry point โ dispatches subcommands, importing each lazily so `count` never loads torch |
|
|
133
|
+
| `src/aidetect/text.py` | shared, torch-free: what counts as prose, what ends a document, how a `.docx` is read |
|
|
134
|
+
| `src/aidetect/count.py` | the IB word count โ sections, citation stripping, budget |
|
|
135
|
+
| `src/aidetect/detect.py` | loads the desklib model, scores each paragraph, prints the bars and flags |
|
|
136
|
+
| `src/aidetect/extract.py` | pulls clean prose out of a `.docx` into a `.txt` โ drops headings, bullets, note-labels |
|
|
137
|
+
| `src/aidetect/binoculars.py` | training-free perplexity-ratio scorer over a base+instruct LM pair (Qwen, or Gemma 4 via `--mlx`; see below) |
|
|
138
|
+
| `src/aidetect/calibrate.py` | fits a threshold on a labelled set you supply, saves it to `~/.config/aidetect` |
|
|
139
|
+
| `src/aidetect/paths.py` | where thresholds are looked up โ `~/.config/aidetect` first, then the ones in the package |
|
|
140
|
+
| `src/aidetect/thresholds/` | the thresholds shipped with the package; a threshold you fit yourself wins over these |
|
|
141
|
+
| `corpora/` | my labelled calibration sets. Repo-only, deliberately not shipped in the package |
|
|
142
|
+
| `tests/` | count rules and Binoculars math, both self-checking, no model download |
|
|
143
|
+
| `pyproject.toml` | package metadata and dependencies โ torch ยท transformers ยท python-docx, plus mlx-vlm on Apple Silicon |
|
|
144
|
+
|
|
145
|
+
## ๐ญ Binoculars: shelved, then revived by Gemma 4
|
|
146
|
+
|
|
147
|
+
[Binoculars](https://arxiv.org/abs/2401.12070) is a training-free detector: run
|
|
148
|
+
text through two LMs that share a tokenizer (a base "observer" and an instruct
|
|
149
|
+
"performer") and divide perplexity by cross-perplexity. Its designed model pair
|
|
150
|
+
is Falcon-7B ร2 (~28GB) โ too big for an 18GB Mac, so the fallback was a small
|
|
151
|
+
same-family pair (Qwen2.5-0.5B or 1.5B) that fits.
|
|
152
|
+
|
|
153
|
+
`aidetect calibrate` scores a labelled set โ mine is 12 real pre-2020 IB Extended
|
|
154
|
+
Essay paragraphs vs 12 LLM-written ones on the same topics โ and finds the best
|
|
155
|
+
separating threshold. Measured across pairs:
|
|
156
|
+
|
|
157
|
+
| pair | best separation | chance |
|
|
158
|
+
|---|---|---|
|
|
159
|
+
| Qwen2.5-0.5B | 62% | 50% |
|
|
160
|
+
| Qwen2.5-1.5B | 67% | 50% |
|
|
161
|
+
| **Gemma 4 E2B** | **96%** | 50% |
|
|
162
|
+
|
|
163
|
+
The Qwen pairs sit near a coin flip: their human and AI score clusters almost
|
|
164
|
+
completely overlap, because the perplexity gap Binoculars exploits is sharp in
|
|
165
|
+
larger models and mush in sub-2B ones. That was the original negative result.
|
|
166
|
+
Swapping in a **Gemma 4** pair opens a clean gap (human mean 0.90 vs AI 0.71)
|
|
167
|
+
and separates the set at 96%. Gemma 4 ships as a multimodal checkpoint, so
|
|
168
|
+
`--mlx` quantizes it to 4-bit and runs it text-only through
|
|
169
|
+
[mlx-vlm](https://github.com/Blaizzy/mlx-vlm), fitting the 18GB Mac in ~6GB:
|
|
170
|
+
|
|
171
|
+
```bash
|
|
172
|
+
aidetect bino IA-clean.txt --mlx --pair gemma # uses the shipped threshold
|
|
173
|
+
|
|
174
|
+
# refit the threshold on your own labelled set
|
|
175
|
+
aidetect calibrate --human-dir corpora/human --ai-dir corpora/ai --mlx --pair gemma
|
|
176
|
+
```
|
|
177
|
+
|
|
178
|
+
desklib stays the primary detector; Binoculars is now a usable second opinion
|
|
179
|
+
rather than a dead end. The calibration sets are not shipped with the package โ
|
|
180
|
+
clone the repo to reproduce the numbers, or point `--human-dir`/`--ai-dir` at
|
|
181
|
+
your own. Your fitted threshold lands in `~/.config/aidetect` and takes
|
|
182
|
+
precedence over the shipped one, so it survives an upgrade.
|
|
183
|
+
|
|
184
|
+
## ๐งช The peer set: genre context, not a human class
|
|
185
|
+
|
|
186
|
+
The 12 human samples are Extended Essays: English, History, Biology, Physics,
|
|
187
|
+
Philosophy. A CS IA is a different animal, all database schemas, GUI components and
|
|
188
|
+
method-by-method justification, and technical prose is inherently more
|
|
189
|
+
predictable token-by-token, which drags perplexity-ratio scores down no matter
|
|
190
|
+
who typed it. So a CS IA scoring below the EE human mean means less than it
|
|
191
|
+
looks.
|
|
192
|
+
|
|
193
|
+
`corpora/peer/` holds 12 paragraphs of real IB Computer Science IA prose
|
|
194
|
+
(5 projects, 5 authors: sudokuMaster, IBOrganizer, MyCalendar, and two
|
|
195
|
+
IBO-published new-syllabus specimens). Measured against the same anchors:
|
|
196
|
+
|
|
197
|
+
| set | Binoculars mean | desklib mean |
|
|
198
|
+
|---|---|---|
|
|
199
|
+
| human (2008 EEs, verified pre-2020) | **0.90** | n/a |
|
|
200
|
+
| **peer (CS IAs, 2021โ2025)** | **0.85** | **0.46** |
|
|
201
|
+
| ai (LLM-written, matched topics) | **0.71** | n/a |
|
|
202
|
+
|
|
203
|
+
The genre gap is real and it is about 0.05 on Binoculars. Score your IA against
|
|
204
|
+
`peer`, not against `human`.
|
|
205
|
+
|
|
206
|
+
**It is not a human class and it never fits a threshold.** Every source
|
|
207
|
+
postdates ChatGPT; three of the five were written in 2025. None carries an
|
|
208
|
+
authorship attestation, and in 2025 a fair share of student IAs were not
|
|
209
|
+
written unaided. Fold that into `human/` and any AI-assisted sample drags the
|
|
210
|
+
mean down, lowers the threshold, and the tool starts clearing drafts for the
|
|
211
|
+
wrong reason, a detector that reassures instead of measures. `aidetect calibrate`
|
|
212
|
+
reads only the two folders you name, so `peer/` stays out of threshold fitting
|
|
213
|
+
by construction, not by discipline.
|
|
214
|
+
|
|
215
|
+
What it can tell you: *"my prose scores like other IAs in this genre."* What it
|
|
216
|
+
can never tell you: *"my prose is human."* Matching a set you cannot vouch for
|
|
217
|
+
proves you are not an outlier, nothing more.
|
|
218
|
+
|
|
219
|
+
**Stack:** python ยท pytorch ยท transformers ยท mlx-vlm ยท desklib DeBERTa
|
|
220
|
+
|
|
221
|
+
The lighter cross-check tool lives at [Ejhfast/fast-ai-detector] โ it's a
|
|
222
|
+
separate repo, not vendored here.
|
|
223
|
+
|
|
224
|
+
---
|
|
225
|
+
|
|
226
|
+
<div align="center">
|
|
227
|
+
|
|
228
|
+
**[Nick Trimandylis](https://github.com/nitrimandylis)**
|
|
229
|
+
|
|
230
|
+
`I WRITE MY OWN ESSAYS โ THIS JUST CHECKS THEY STILL READ LIKE IT`
|
|
231
|
+
|
|
232
|
+
MIT licensed โ see [LICENSE](LICENSE).
|
|
233
|
+
|
|
234
|
+
</div>
|
|
235
|
+
|
|
236
|
+
[Ejhfast/fast-ai-detector]: https://github.com/Ejhfast/fast-ai-detector
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
# aidetect
|
|
2
|
+
|
|
3
|
+
Local, offline AI-writing checks and IB word counts for my own drafts (EE, IA,
|
|
4
|
+
etc). Tells me whether my prose reads as AI-generated *before* a teacher runs
|
|
5
|
+
Turnitin, and whether the draft is over its word limit before I hand it in.
|
|
6
|
+
|
|
7
|
+
Packaged for PyPI as `aidetect` so it installs rather than being cloned. The
|
|
8
|
+
0.1.0 wheel is built and verified against a clean venv; the upload itself has
|
|
9
|
+
not happened yet.
|
|
10
|
+
|
|
11
|
+
```bash
|
|
12
|
+
uv tool install aidetect
|
|
13
|
+
```
|
|
14
|
+
|
|
15
|
+
## What it is
|
|
16
|
+
|
|
17
|
+
One command, five subcommands. Subcommand modules import lazily, so the two
|
|
18
|
+
that need no model stay instant.
|
|
19
|
+
|
|
20
|
+
- `aidetect count` โ the IB word count. Excludes headings, block quotes,
|
|
21
|
+
bullets, tables, footnotes, in-text citations and everything from the
|
|
22
|
+
bibliography on, then reports the total **by section** against `--limit`.
|
|
23
|
+
Word's own count is wrong for IB and does not say which section is bloated.
|
|
24
|
+
`--json` emits the whole thing as one object for scripts and agents.
|
|
25
|
+
- `aidetect score` โ the **desklib/ai-text-detector-v1.01** model (DeBERTa, #1
|
|
26
|
+
on the RAID benchmark). Prints an AI score (0โ1) per paragraph, flagging
|
|
27
|
+
anything >= 0.5.
|
|
28
|
+
- `aidetect bino` โ training-free third scorer (base + instruct LM pair,
|
|
29
|
+
perplexity รท cross-perplexity). Shelved with the Qwen pairs (62โ67% on the
|
|
30
|
+
labelled set, near chance) but **revived by Gemma 4**: the E2B pair separates
|
|
31
|
+
the same set at 96%. Gemma 4 is a multimodal checkpoint, so `--mlx` loads it
|
|
32
|
+
4-bit via mlx-vlm and runs it text-only, fitting an 18GB Mac in ~6GB. desklib
|
|
33
|
+
stays primary; Binoculars is a usable second opinion rather than a dead end.
|
|
34
|
+
- `aidetect extract` โ strips a `.docx` down to finished prose for the scorers.
|
|
35
|
+
- `aidetect calibrate` โ refits a Binoculars threshold on a labelled set you
|
|
36
|
+
name via `--human-dir`/`--ai-dir`, and writes it to `~/.config/aidetect`.
|
|
37
|
+
- `fast-ai-detector/` โ a separate, lighter tool ([Ejhfast/fast-ai-detector])
|
|
38
|
+
to cross-check. Not vendored, gitignored, its own CLI.
|
|
39
|
+
|
|
40
|
+
## Decisions worth remembering
|
|
41
|
+
|
|
42
|
+
- **Thresholds are two-tier.** The wheel ships the fitted ones; anything you
|
|
43
|
+
refit lands in `~/.config/aidetect` and wins. Calibration output never writes
|
|
44
|
+
into site-packages, which is replaced on upgrade.
|
|
45
|
+
- **`corpora/` does not ship.** It is 12 real students' Extended Essays plus
|
|
46
|
+
matched LLM imitations. Fine in a repo, not something to push through PyPI on
|
|
47
|
+
every install. `calibrate` therefore requires explicit folder arguments.
|
|
48
|
+
- **`text.py` is torch-free on purpose.** `count` must not pay a ~2s model
|
|
49
|
+
import to count words, and the split is enforced by a check in `tests/`.
|
|
50
|
+
- **The MLX cache stays at `~/.cache/ai-detect-mlx`** despite the rename.
|
|
51
|
+
Moving it would force a 6GB re-download and re-quantization for nothing.
|
|
52
|
+
|
|
53
|
+
## Honest limits
|
|
54
|
+
|
|
55
|
+
- Runs on an 18GB Mac (Apple Silicon MPS). Binoculars' *default* Falcon-7B pair
|
|
56
|
+
(~28GB) won't fit; the Qwen pairs fit but barely separate, and the Gemma 4
|
|
57
|
+
MLX pair is the one that actually works. Fast-DetectGPT skipped.
|
|
58
|
+
- Directional only. A high score means "reword this," not "you'll get caught."
|
|
59
|
+
It is not, and cannot be, the Turnitin number a teacher sees.
|
|
60
|
+
- `count` implements the exclusions the EE and the subject IAs share. Per-subject
|
|
61
|
+
quirks are not encoded; check your own subject guide for the limit itself.
|
|
62
|
+
- Citation stripping keys on a year, `ibid` or `et al` inside parentheses. A
|
|
63
|
+
citation style that uses none of those is not detected and will be counted.
|
|
64
|
+
|
|
65
|
+
[Ejhfast/fast-ai-detector]: https://github.com/Ejhfast/fast-ai-detector
|
aidetect-0.1.0/README.md
ADDED
|
@@ -0,0 +1,216 @@
|
|
|
1
|
+
```
|
|
2
|
+
โโโโโโ โโโ โโโโโโโ โโโโโโโโโโโโโโโโโโโโโโโโโ โโโโโโโโโโโโโโโโ
|
|
3
|
+
โโโโโโโโโโโ โโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโ
|
|
4
|
+
โโโโโโโโโโโโโโโโโโโโ โโโโโโโโโ โโโ โโโโโโ โโโ โโโ
|
|
5
|
+
โโโโโโโโโโโโโโโโโโโโ โโโโโโโโโ โโโ โโโโโโ โโโ โโโ
|
|
6
|
+
โโโ โโโโโโ โโโโโโโโโโโโโโโโ โโโ โโโโโโโโโโโโโโโโ โโโ
|
|
7
|
+
โโโ โโโโโโ โโโโโโโ โโโโโโโโ โโโ โโโโโโโโ โโโโโโโ โโโ
|
|
8
|
+
```
|
|
9
|
+
<div align="center">
|
|
10
|
+
|
|
11
|
+
### `SCORE YOUR OWN PROSE // BEFORE A TEACHER SCORES IT FOR YOU`
|
|
12
|
+
|
|
13
|
+
*a local, offline AI-writing detector and IB word counter for drafts you actually wrote*
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
-111111?style=flat-square&labelColor=111111)
|
|
21
|
+
|
|
22
|
+
</div>
|
|
23
|
+
|
|
24
|
+
---
|
|
25
|
+
|
|
26
|
+
## ๐ What is this
|
|
27
|
+
|
|
28
|
+
A command-line tool that reads a `.docx` or `.txt` and scores each paragraph
|
|
29
|
+
0โ1 on how AI-generated it reads, using the `desklib/ai-text-detector-v1.01`
|
|
30
|
+
DeBERTa model โ the one sitting at #1 on the RAID benchmark. Everything runs
|
|
31
|
+
on your own machine; after the first model download it never touches the
|
|
32
|
+
network. You point it at your Extended Essay, it tells you which paragraphs
|
|
33
|
+
sound like a language model wrote them.
|
|
34
|
+
|
|
35
|
+
The point isn't to cheat a detector. It's the opposite: I write my own drafts,
|
|
36
|
+
and sometimes my own honest prose still trips these classifiers because that's
|
|
37
|
+
what earnest formal writing looks like to them. This flags those paragraphs so
|
|
38
|
+
I can reword before a teacher runs Turnitin and has an awkward conversation
|
|
39
|
+
with me about it.
|
|
40
|
+
|
|
41
|
+
It is directional, not oracular. A high score means "reword this," not "you're
|
|
42
|
+
caught." It is not, and cannot be, the number Turnitin shows a teacher.
|
|
43
|
+
|
|
44
|
+
```console
|
|
45
|
+
nick@aidetect:~$ aidetect score EE-clean.txt
|
|
46
|
+
P 1 0.08 [##------------------]
|
|
47
|
+
P 2 0.71 [##############------] <-- AI-ish
|
|
48
|
+
average AI score: 0.34 | 1/6 paragraphs flagged
|
|
49
|
+
reminder: directional only, not a Turnitin score.
|
|
50
|
+
```
|
|
51
|
+
|
|
52
|
+
## ๐ง The detection engine
|
|
53
|
+
|
|
54
|
+
| | feature | what it actually does |
|
|
55
|
+
|---|---|---|
|
|
56
|
+
| 01 | **per-paragraph scoring** | what it actually catches โ splits your draft and scores each paragraph, so you fix the two bad ones instead of rewriting everything |
|
|
57
|
+
| 02 | **docx + txt input** | reads Word files straight (paragraphs, no headings) or plain text split on blank lines |
|
|
58
|
+
| 03 | **prose extractor** | `aidetect extract` strips headings, bullets, footnotes and your own note-scaffolding first, so the score is about writing, not structure |
|
|
59
|
+
| 04 | **offline after setup** | first run pulls ~1.5GB of model, every run after is airgapped โ your essay never leaves the laptop |
|
|
60
|
+
| 05 | **second opinion** | cross-check against the lighter [Ejhfast/fast-ai-detector] when one model's paranoia isn't enough |
|
|
61
|
+
| 06 | **Binoculars (Gemma 4)** | a training-free perplexity-ratio detector โ near chance with small Qwen pairs, but 96% on the labelled set once swapped to a Gemma 4 pair; see below |
|
|
62
|
+
| 07 | **IB word count** | `aidetect count` (`--json` for scripts and agents) counts what the IB counts โ no headings, quotes, tables, footnotes, citations or bibliography โ and splits the total by section, so an over-long draft tells you *where* |
|
|
63
|
+
|
|
64
|
+
## ๐ Run it
|
|
65
|
+
|
|
66
|
+
```bash
|
|
67
|
+
uv tool install aidetect # or: pipx install aidetect
|
|
68
|
+
```
|
|
69
|
+
|
|
70
|
+
```bash
|
|
71
|
+
aidetect # list the five subcommands
|
|
72
|
+
aidetect count "draft.docx" --limit 4000 # IB word count, by section
|
|
73
|
+
aidetect count "draft.docx" --json # same, as one JSON object
|
|
74
|
+
aidetect score "draft.docx" # score a whole draft
|
|
75
|
+
aidetect score --text "one sentence" # score a single string
|
|
76
|
+
aidetect bino "draft.docx" --mlx --pair gemma
|
|
77
|
+
```
|
|
78
|
+
|
|
79
|
+
`count` and `extract` are instant and need no model. `score` and `bino` need a
|
|
80
|
+
machine that can hold a transformer: built and tested on an 18GB Apple Silicon
|
|
81
|
+
Mac, MPS-accelerated. Their first run downloads the model and will sit there for
|
|
82
|
+
a minute โ that's normal, not a hang. Every run after is fast and offline.
|
|
83
|
+
|
|
84
|
+
On Apple Silicon the Gemma 4 MLX pair installs automatically. Elsewhere it is
|
|
85
|
+
skipped and the Qwen pairs still work.
|
|
86
|
+
|
|
87
|
+
### `--json`
|
|
88
|
+
|
|
89
|
+
`count` takes `--json` and prints exactly one object on stdout, nothing else:
|
|
90
|
+
|
|
91
|
+
```json
|
|
92
|
+
{"sections": [{"title": "Introduction", "words": 812}], "total": 3940, "limit": 4000, "over": -60}
|
|
93
|
+
```
|
|
94
|
+
|
|
95
|
+
Every key is always present. `limit` and `over` are `null` when no `--limit` was
|
|
96
|
+
given โ that means "does not apply", not "could not be read". A draft with no
|
|
97
|
+
prose is an empty `sections` list and exit 0. Errors go to stderr with a non-zero
|
|
98
|
+
exit, so consumers branch on the exit code rather than parsing error text.
|
|
99
|
+
|
|
100
|
+
## ๐ฉ Under the hood
|
|
101
|
+
|
|
102
|
+
```mermaid
|
|
103
|
+
flowchart LR
|
|
104
|
+
A[.docx / .txt] --> B[aidetect extract<br/>strip non-prose]
|
|
105
|
+
B --> C[read_paragraphs<br/>>= 25 words]
|
|
106
|
+
C --> D[desklib DeBERTa<br/>mean-pool + sigmoid]
|
|
107
|
+
D --> E[per-paragraph<br/>0-1 score + flags]
|
|
108
|
+
```
|
|
109
|
+
|
|
110
|
+
| file | job |
|
|
111
|
+
|---|---|
|
|
112
|
+
| `src/aidetect/cli.py` | the `aidetect` entry point โ dispatches subcommands, importing each lazily so `count` never loads torch |
|
|
113
|
+
| `src/aidetect/text.py` | shared, torch-free: what counts as prose, what ends a document, how a `.docx` is read |
|
|
114
|
+
| `src/aidetect/count.py` | the IB word count โ sections, citation stripping, budget |
|
|
115
|
+
| `src/aidetect/detect.py` | loads the desklib model, scores each paragraph, prints the bars and flags |
|
|
116
|
+
| `src/aidetect/extract.py` | pulls clean prose out of a `.docx` into a `.txt` โ drops headings, bullets, note-labels |
|
|
117
|
+
| `src/aidetect/binoculars.py` | training-free perplexity-ratio scorer over a base+instruct LM pair (Qwen, or Gemma 4 via `--mlx`; see below) |
|
|
118
|
+
| `src/aidetect/calibrate.py` | fits a threshold on a labelled set you supply, saves it to `~/.config/aidetect` |
|
|
119
|
+
| `src/aidetect/paths.py` | where thresholds are looked up โ `~/.config/aidetect` first, then the ones in the package |
|
|
120
|
+
| `src/aidetect/thresholds/` | the thresholds shipped with the package; a threshold you fit yourself wins over these |
|
|
121
|
+
| `corpora/` | my labelled calibration sets. Repo-only, deliberately not shipped in the package |
|
|
122
|
+
| `tests/` | count rules and Binoculars math, both self-checking, no model download |
|
|
123
|
+
| `pyproject.toml` | package metadata and dependencies โ torch ยท transformers ยท python-docx, plus mlx-vlm on Apple Silicon |
|
|
124
|
+
|
|
125
|
+
## ๐ญ Binoculars: shelved, then revived by Gemma 4
|
|
126
|
+
|
|
127
|
+
[Binoculars](https://arxiv.org/abs/2401.12070) is a training-free detector: run
|
|
128
|
+
text through two LMs that share a tokenizer (a base "observer" and an instruct
|
|
129
|
+
"performer") and divide perplexity by cross-perplexity. Its designed model pair
|
|
130
|
+
is Falcon-7B ร2 (~28GB) โ too big for an 18GB Mac, so the fallback was a small
|
|
131
|
+
same-family pair (Qwen2.5-0.5B or 1.5B) that fits.
|
|
132
|
+
|
|
133
|
+
`aidetect calibrate` scores a labelled set โ mine is 12 real pre-2020 IB Extended
|
|
134
|
+
Essay paragraphs vs 12 LLM-written ones on the same topics โ and finds the best
|
|
135
|
+
separating threshold. Measured across pairs:
|
|
136
|
+
|
|
137
|
+
| pair | best separation | chance |
|
|
138
|
+
|---|---|---|
|
|
139
|
+
| Qwen2.5-0.5B | 62% | 50% |
|
|
140
|
+
| Qwen2.5-1.5B | 67% | 50% |
|
|
141
|
+
| **Gemma 4 E2B** | **96%** | 50% |
|
|
142
|
+
|
|
143
|
+
The Qwen pairs sit near a coin flip: their human and AI score clusters almost
|
|
144
|
+
completely overlap, because the perplexity gap Binoculars exploits is sharp in
|
|
145
|
+
larger models and mush in sub-2B ones. That was the original negative result.
|
|
146
|
+
Swapping in a **Gemma 4** pair opens a clean gap (human mean 0.90 vs AI 0.71)
|
|
147
|
+
and separates the set at 96%. Gemma 4 ships as a multimodal checkpoint, so
|
|
148
|
+
`--mlx` quantizes it to 4-bit and runs it text-only through
|
|
149
|
+
[mlx-vlm](https://github.com/Blaizzy/mlx-vlm), fitting the 18GB Mac in ~6GB:
|
|
150
|
+
|
|
151
|
+
```bash
|
|
152
|
+
aidetect bino IA-clean.txt --mlx --pair gemma # uses the shipped threshold
|
|
153
|
+
|
|
154
|
+
# refit the threshold on your own labelled set
|
|
155
|
+
aidetect calibrate --human-dir corpora/human --ai-dir corpora/ai --mlx --pair gemma
|
|
156
|
+
```
|
|
157
|
+
|
|
158
|
+
desklib stays the primary detector; Binoculars is now a usable second opinion
|
|
159
|
+
rather than a dead end. The calibration sets are not shipped with the package โ
|
|
160
|
+
clone the repo to reproduce the numbers, or point `--human-dir`/`--ai-dir` at
|
|
161
|
+
your own. Your fitted threshold lands in `~/.config/aidetect` and takes
|
|
162
|
+
precedence over the shipped one, so it survives an upgrade.
|
|
163
|
+
|
|
164
|
+
## ๐งช The peer set: genre context, not a human class
|
|
165
|
+
|
|
166
|
+
The 12 human samples are Extended Essays: English, History, Biology, Physics,
|
|
167
|
+
Philosophy. A CS IA is a different animal, all database schemas, GUI components and
|
|
168
|
+
method-by-method justification, and technical prose is inherently more
|
|
169
|
+
predictable token-by-token, which drags perplexity-ratio scores down no matter
|
|
170
|
+
who typed it. So a CS IA scoring below the EE human mean means less than it
|
|
171
|
+
looks.
|
|
172
|
+
|
|
173
|
+
`corpora/peer/` holds 12 paragraphs of real IB Computer Science IA prose
|
|
174
|
+
(5 projects, 5 authors: sudokuMaster, IBOrganizer, MyCalendar, and two
|
|
175
|
+
IBO-published new-syllabus specimens). Measured against the same anchors:
|
|
176
|
+
|
|
177
|
+
| set | Binoculars mean | desklib mean |
|
|
178
|
+
|---|---|---|
|
|
179
|
+
| human (2008 EEs, verified pre-2020) | **0.90** | n/a |
|
|
180
|
+
| **peer (CS IAs, 2021โ2025)** | **0.85** | **0.46** |
|
|
181
|
+
| ai (LLM-written, matched topics) | **0.71** | n/a |
|
|
182
|
+
|
|
183
|
+
The genre gap is real and it is about 0.05 on Binoculars. Score your IA against
|
|
184
|
+
`peer`, not against `human`.
|
|
185
|
+
|
|
186
|
+
**It is not a human class and it never fits a threshold.** Every source
|
|
187
|
+
postdates ChatGPT; three of the five were written in 2025. None carries an
|
|
188
|
+
authorship attestation, and in 2025 a fair share of student IAs were not
|
|
189
|
+
written unaided. Fold that into `human/` and any AI-assisted sample drags the
|
|
190
|
+
mean down, lowers the threshold, and the tool starts clearing drafts for the
|
|
191
|
+
wrong reason, a detector that reassures instead of measures. `aidetect calibrate`
|
|
192
|
+
reads only the two folders you name, so `peer/` stays out of threshold fitting
|
|
193
|
+
by construction, not by discipline.
|
|
194
|
+
|
|
195
|
+
What it can tell you: *"my prose scores like other IAs in this genre."* What it
|
|
196
|
+
can never tell you: *"my prose is human."* Matching a set you cannot vouch for
|
|
197
|
+
proves you are not an outlier, nothing more.
|
|
198
|
+
|
|
199
|
+
**Stack:** python ยท pytorch ยท transformers ยท mlx-vlm ยท desklib DeBERTa
|
|
200
|
+
|
|
201
|
+
The lighter cross-check tool lives at [Ejhfast/fast-ai-detector] โ it's a
|
|
202
|
+
separate repo, not vendored here.
|
|
203
|
+
|
|
204
|
+
---
|
|
205
|
+
|
|
206
|
+
<div align="center">
|
|
207
|
+
|
|
208
|
+
**[Nick Trimandylis](https://github.com/nitrimandylis)**
|
|
209
|
+
|
|
210
|
+
`I WRITE MY OWN ESSAYS โ THIS JUST CHECKS THEY STILL READ LIKE IT`
|
|
211
|
+
|
|
212
|
+
MIT licensed โ see [LICENSE](LICENSE).
|
|
213
|
+
|
|
214
|
+
</div>
|
|
215
|
+
|
|
216
|
+
[Ejhfast/fast-ai-detector]: https://github.com/Ejhfast/fast-ai-detector
|