slopscore 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- slopscore-0.1.0/LICENSE +21 -0
- slopscore-0.1.0/PKG-INFO +299 -0
- slopscore-0.1.0/README.md +278 -0
- slopscore-0.1.0/pyproject.toml +41 -0
- slopscore-0.1.0/setup.cfg +4 -0
- slopscore-0.1.0/src/slopscore/__init__.py +3 -0
- slopscore-0.1.0/src/slopscore/cli.py +283 -0
- slopscore-0.1.0/src/slopscore/config.py +76 -0
- slopscore-0.1.0/src/slopscore/githooks.py +319 -0
- slopscore-0.1.0/src/slopscore/imports.py +139 -0
- slopscore-0.1.0/src/slopscore/ingest.py +46 -0
- slopscore-0.1.0/src/slopscore/signals.py +659 -0
- slopscore-0.1.0/src/slopscore/triage.py +273 -0
- slopscore-0.1.0/src/slopscore.egg-info/PKG-INFO +299 -0
- slopscore-0.1.0/src/slopscore.egg-info/SOURCES.txt +28 -0
- slopscore-0.1.0/src/slopscore.egg-info/dependency_links.txt +1 -0
- slopscore-0.1.0/src/slopscore.egg-info/entry_points.txt +2 -0
- slopscore-0.1.0/src/slopscore.egg-info/top_level.txt +1 -0
- slopscore-0.1.0/tests/test_calibration.py +123 -0
- slopscore-0.1.0/tests/test_cli.py +288 -0
- slopscore-0.1.0/tests/test_code_signals.py +65 -0
- slopscore-0.1.0/tests/test_config.py +97 -0
- slopscore-0.1.0/tests/test_corpus.py +77 -0
- slopscore-0.1.0/tests/test_githooks.py +70 -0
- slopscore-0.1.0/tests/test_holdout.py +51 -0
- slopscore-0.1.0/tests/test_hooks.py +318 -0
- slopscore-0.1.0/tests/test_imports.py +98 -0
- slopscore-0.1.0/tests/test_ingest.py +48 -0
- slopscore-0.1.0/tests/test_signals.py +215 -0
- slopscore-0.1.0/tests/test_triage.py +196 -0
slopscore-0.1.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Matt Whelan
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
slopscore-0.1.0/PKG-INFO
ADDED
|
@@ -0,0 +1,299 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: slopscore
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Heuristic linter that scores AI residue in commits and PRs 0-100 with evidence - flags craft, not authorship
|
|
5
|
+
Author: Matt Whelan
|
|
6
|
+
License: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/koopatroopa/slopscore
|
|
8
|
+
Project-URL: Repository, https://github.com/koopatroopa/slopscore
|
|
9
|
+
Project-URL: Issues, https://github.com/koopatroopa/slopscore/issues
|
|
10
|
+
Keywords: lint,ai,slop,commit,git-hooks,pull-request,code-quality,ci
|
|
11
|
+
Classifier: Development Status :: 4 - Beta
|
|
12
|
+
Classifier: Environment :: Console
|
|
13
|
+
Classifier: Intended Audience :: Developers
|
|
14
|
+
Classifier: Operating System :: OS Independent
|
|
15
|
+
Classifier: Programming Language :: Python :: 3
|
|
16
|
+
Classifier: Topic :: Software Development :: Quality Assurance
|
|
17
|
+
Requires-Python: >=3.11
|
|
18
|
+
Description-Content-Type: text/markdown
|
|
19
|
+
License-File: LICENSE
|
|
20
|
+
Dynamic: license-file
|
|
21
|
+
|
|
22
|
+
# slopscore
|
|
23
|
+
|
|
24
|
+
[](https://github.com/koopatroopa/slopscore/actions/workflows/ci.yml)
|
|
25
|
+
[](https://pypi.org/project/slopscore/)
|
|
26
|
+
[](https://www.python.org/downloads/)
|
|
27
|
+
[](LICENSE)
|
|
28
|
+
|
|
29
|
+
### **_Catch the slop before it ships._**
|
|
30
|
+
|
|
31
|
+
AI tools leave fingerprints — em-dashes, a rocket 🚀 emoji no human would
|
|
32
|
+
ever type by hand, stray `Co-Authored-By: Claude` trailers,
|
|
33
|
+
`# ... rest of the code unchanged` stubs, ghost import declarations,
|
|
34
|
+
`Summary by CodeRabbit` stamps. slopscore detects these AI-generated
|
|
35
|
+
tells by putting a number on them — every commit, push and PR scored out
|
|
36
|
+
of 100, every finding backed by evidence, all before it ships.
|
|
37
|
+
|
|
38
|
+
## Install
|
|
39
|
+
|
|
40
|
+
```sh
|
|
41
|
+
pip install slopscore
|
|
42
|
+
```
|
|
43
|
+
|
|
44
|
+
Then, inside each repo you want watched:
|
|
45
|
+
|
|
46
|
+
```sh
|
|
47
|
+
slopscore install-hooks
|
|
48
|
+
```
|
|
49
|
+
|
|
50
|
+
That second step is the point of the tool - a score on every commit and push.
|
|
51
|
+
Python 3.11 or newer. No other dependencies.
|
|
52
|
+
|
|
53
|
+
Plenty of tools lint AI residue in code. **Nothing else scores the prose that
|
|
54
|
+
ships with it** - the commit messages and PR text where the residue lives in
|
|
55
|
+
git history forever. It starts as a nudge; **one git config turns it into a
|
|
56
|
+
hard gate** that refuses any commit or push over your threshold, and off again
|
|
57
|
+
just as fast.
|
|
58
|
+
|
|
59
|
+
It flags **craft, not authorship**. slopscore never claims "this is AI" - it
|
|
60
|
+
points at fixable leftovers with evidence, like a linter flagging a missing
|
|
61
|
+
semicolon. (Style-based AI detection is unreliable and biased against
|
|
62
|
+
second-language writers, so it refuses to do it: 24 real pre-2020 human PRs
|
|
63
|
+
must score LOW forever - `tests/test_holdout.py`.) Everything runs locally: no
|
|
64
|
+
LLM, no network, no telemetry.
|
|
65
|
+
|
|
66
|
+
## Who runs it
|
|
67
|
+
|
|
68
|
+
Two kinds of people, both on their **own** work:
|
|
69
|
+
|
|
70
|
+
- **You used AI and want to ship clean.** Catch the residue - a leftover
|
|
71
|
+
trailer, a placeholder stub, unedited boilerplate - before a reviewer does.
|
|
72
|
+
Run it on yourself via the hooks, or as a shared standard in CI (report-only
|
|
73
|
+
for repos with outside contributors).
|
|
74
|
+
- **You didn't use AI but fear a detector will say you did.** Formal and
|
|
75
|
+
second-language writers get wrongly flagged constantly. Turn on the thorough
|
|
76
|
+
tier (`--strict`) and slopscore shows your *own* writing through a crude
|
|
77
|
+
detector's eyes - the em-dashes, the "delve", the scaffolding those tools
|
|
78
|
+
latch onto - with evidence, so you can see your exposure and decide whether
|
|
79
|
+
to defang it. slopscore does not believe these prove AI; it refuses that. It
|
|
80
|
+
just lets you see what the flawed tools out there react to.
|
|
81
|
+
|
|
82
|
+
Either way the score is *information*, never an accusation: the verdict is the
|
|
83
|
+
only gate, and honest human work is built to pass.
|
|
84
|
+
|
|
85
|
+
## Tested against reality
|
|
86
|
+
|
|
87
|
+
Every claim above is measured, not asserted. The signals are validated
|
|
88
|
+
against nearly 5,000 real commits and PR bodies from public GitHub history,
|
|
89
|
+
under pre-registered rules: the metric and the pass bar are written down
|
|
90
|
+
before the data is scored, every rejected idea is logged, and a held-out set
|
|
91
|
+
of 24 pre-2020 human PRs is never tuned against - CI enforces that it scores
|
|
92
|
+
LOW forever.
|
|
93
|
+
|
|
94
|
+
- **2,300+ real human commits and PR bodies (pre-2022, definitionally
|
|
95
|
+
pre-LLM): zero false flags.**
|
|
96
|
+
- **1,700+ commits and PR bodies carrying real AI attribution: 99.9%
|
|
97
|
+
flagged.**
|
|
98
|
+
- **785 disciplined, human-reviewed AI-assisted commits: all PASS.** It
|
|
99
|
+
measures residue, not authorship - AI-assisted work that someone actually
|
|
100
|
+
edited scores like human work.
|
|
101
|
+
|
|
102
|
+
When the corpus catches the engine being wrong, the engine changes - and
|
|
103
|
+
signals that false-fired on real humans were rejected and stay rejected.
|
|
104
|
+
|
|
105
|
+
## Try it
|
|
106
|
+
|
|
107
|
+
Scoring a commit on its way out - the message and the staged code, with each
|
|
108
|
+
finding pinned to where it is:
|
|
109
|
+
|
|
110
|
+

|
|
111
|
+
|
|
112
|
+
Or score raw text from the command line:
|
|
113
|
+
|
|
114
|
+
```sh
|
|
115
|
+
echo "Certainly! Let's delve into a robust refactor. Generated with Claude Code." | slopscore --text -
|
|
116
|
+
```
|
|
117
|
+
|
|
118
|
+

|
|
119
|
+
|
|
120
|
+
Drop the "Generated with Claude Code." sentence and it falls to 13.6, PASS -
|
|
121
|
+
single weak signals are normal writing; only convergence flags.
|
|
122
|
+
|
|
123
|
+
The score is a **gradient** - *how much* AI residue is in the text, not a
|
|
124
|
+
yes/no. A low non-zero score is light texture, not an accusation; the
|
|
125
|
+
**verdict** is the only gate, and it is tuned so honest human work passes.
|
|
126
|
+
|
|
127
|
+
- Bands: **LOW** below 30, **MEDIUM** 30 to under 70, **HIGH** 70 and up. The
|
|
128
|
+
**verdict** FLAGs at/above the threshold (default 30), so MEDIUM and HIGH
|
|
129
|
+
both flag.
|
|
130
|
+
- The **exit code** follows the verdict: `0` pass, `1` flag, `2` usage error -
|
|
131
|
+
so it drops straight into CI or a git hook.
|
|
132
|
+
- Evidence carries `path:line` for code; JSON input gets prose locations too
|
|
133
|
+
(`body:14`, `commit[2]:1`).
|
|
134
|
+
|
|
135
|
+
On a terminal the report is coloured by band (green/amber/red); piped output
|
|
136
|
+
stays plain. `--color` and `NO_COLOR` override.
|
|
137
|
+
|
|
138
|
+
## Three ways to use it
|
|
139
|
+
|
|
140
|
+
**1. The CLI - check your work by hand.**
|
|
141
|
+
|
|
142
|
+
```sh
|
|
143
|
+
# score a PR/issue described as JSON ({title, body, commits})
|
|
144
|
+
slopscore pr.json
|
|
145
|
+
# or raw text, or stdin
|
|
146
|
+
echo "Quick fix. Generated with Claude Code." | slopscore --text -
|
|
147
|
+
# scan code files too (go wide on your working tree)
|
|
148
|
+
slopscore --files src/*.py --json
|
|
149
|
+
# thorough tier: also score the opt-in signals (see "Who runs it")
|
|
150
|
+
slopscore --strict pr.json
|
|
151
|
+
```
|
|
152
|
+
|
|
153
|
+
**2. Git hooks - score every commit and push; block them when you say so.**
|
|
154
|
+
|
|
155
|
+
```sh
|
|
156
|
+
slopscore install-hooks
|
|
157
|
+
```
|
|
158
|
+
|
|
159
|
+
Or via the [pre-commit](https://pre-commit.com) framework:
|
|
160
|
+
|
|
161
|
+
```yaml
|
|
162
|
+
repos:
|
|
163
|
+
- repo: https://github.com/koopatroopa/slopscore
|
|
164
|
+
rev: v0.1.0
|
|
165
|
+
hooks:
|
|
166
|
+
- id: slopscore-commit-msg
|
|
167
|
+
- id: slopscore-pre-push
|
|
168
|
+
```
|
|
169
|
+
|
|
170
|
+
`commit-msg` scores your message plus the staged code; `pre-push` scores each
|
|
171
|
+
outgoing commit (flagged ones reported as `[abc1234] Subject`, clean pushes
|
|
172
|
+
silent). **Advisory by default - they never block until you ask.** The gate is
|
|
173
|
+
one setting, per repo, instant in both directions:
|
|
174
|
+
|
|
175
|
+
```sh
|
|
176
|
+
git config slopscore.block true # gate ON: refuse commits/pushes at/above the threshold
|
|
177
|
+
git config slopscore.threshold 50 # move the bar (default 30)
|
|
178
|
+
git config slopscore.block false # gate OFF: back to advisory
|
|
179
|
+
git config slopscore.strict true # thorough tier: score the opt-in signals too
|
|
180
|
+
```
|
|
181
|
+
|
|
182
|
+
Escape hatches even with the gate on: `git commit --no-verify` (or push) skips
|
|
183
|
+
it once; `SLOPSCORE_BLOCK=0` (or `=1`) overrides the setting for one command.
|
|
184
|
+
Every report's footer tells you the current state and the command to flip it.
|
|
185
|
+
|
|
186
|
+
**3. CI - the same gate on every PR, two lines on either platform.**
|
|
187
|
+
|
|
188
|
+
Both recipes score the PR/MR's prose (title + description) plus the diff's
|
|
189
|
+
added lines, report-only until you flip the gate (exit `0` pass, `1` flag).
|
|
190
|
+
It runs on every contributor, so it is a shared craft standard - keep it
|
|
191
|
+
report-only on repos with outside contributors.
|
|
192
|
+
|
|
193
|
+
GitHub (run it under `pull_request`, not `pull_request_target` - slopscore
|
|
194
|
+
only needs to read the diff, never repo write access or secrets):
|
|
195
|
+
|
|
196
|
+
```yaml
|
|
197
|
+
on: pull_request
|
|
198
|
+
permissions:
|
|
199
|
+
contents: read
|
|
200
|
+
# ...
|
|
201
|
+
- uses: actions/checkout@v4
|
|
202
|
+
with: { fetch-depth: 0 }
|
|
203
|
+
- uses: koopatroopa/slopscore@v0 # pin to a tag
|
|
204
|
+
with: { fail-on-flag: "false" } # "true" = gate the merge
|
|
205
|
+
```
|
|
206
|
+
|
|
207
|
+
GitLab:
|
|
208
|
+
|
|
209
|
+
```yaml
|
|
210
|
+
include:
|
|
211
|
+
- remote: https://raw.githubusercontent.com/koopatroopa/slopscore/main/ci/slopscore.gitlab-ci.yml
|
|
212
|
+
```
|
|
213
|
+
|
|
214
|
+
The GitLab job ships advisory (`allow_failure: true`); redeclare the job to
|
|
215
|
+
remove that or set `SLOPSCORE_THRESHOLD`. Any other CI works the same way:
|
|
216
|
+
`pip install slopscore`, feed it `--text` and `--diff`, gate on the exit
|
|
217
|
+
code.
|
|
218
|
+
|
|
219
|
+
## What it looks for
|
|
220
|
+
|
|
221
|
+
Signals that are **on by default** - distinctive leftovers with a very low
|
|
222
|
+
false-positive rate:
|
|
223
|
+
|
|
224
|
+
- `ai_self_reference` - explicit AI attribution: trailers ("Co-Authored-By:
|
|
225
|
+
Claude"), assistant self-talk ("As an AI..."), and the stamps AI review
|
|
226
|
+
bots leave in PR bodies ("## Summary by CodeRabbit", aider's
|
|
227
|
+
auto-generated-PR header)
|
|
228
|
+
- `ai_cliche_phrases` - chatbot filler ("delve into", "it's worth noting")
|
|
229
|
+
- `code_placeholder_stub` - placeholder markers left in code ("// ... rest of
|
|
230
|
+
code", "your implementation here"), reported with file and line.
|
|
231
|
+
- `em_dash_density`, `emoji_density` and `curly_quotes` - U+2014, decorative
|
|
232
|
+
emoji and word-processor quotes are not on your keyboard; in coding
|
|
233
|
+
artefacts they arrive via tooling. These only ever add to a score (they are
|
|
234
|
+
excluded from the normalisation ceiling), so enabling or disabling them
|
|
235
|
+
cannot dilute the signals above - and their combined contribution is
|
|
236
|
+
capped, so however many fire at once they can colour a score but never
|
|
237
|
+
flag on their own. Real Greenkeeper-era PR bodies taught us that one.
|
|
238
|
+
|
|
239
|
+
Signals that are **opt-in**, because humans genuinely type them:
|
|
240
|
+
|
|
241
|
+
- `code_undeclared_import` - an import that is not in the standard library, not
|
|
242
|
+
declared in your manifest and not a local module - possibly a package the
|
|
243
|
+
model made up. It reads your `pyproject`/`requirements` and never imports or
|
|
244
|
+
installs anything.
|
|
245
|
+
- `sycophantic_openers` ("Certainly!", "Hope this helps!") - chat register,
|
|
246
|
+
not commit register: across 2,500+ real AI-attributed commits and PR bodies
|
|
247
|
+
it fired zero times, and its only corpus hits were friendly humans. Kept
|
|
248
|
+
for scanning pasted chat output.
|
|
249
|
+
- `promotional_adjectives` ("robust", "comprehensive"), `section_scaffolding`
|
|
250
|
+
(`## Overview` headers - PR templates generate these), `bold_lead_in_lists`,
|
|
251
|
+
`negative_parallelism` ("not just X, but Y" and its TED-talk cousins),
|
|
252
|
+
`rhetorical_qa` ("Why? Because...") and `vague_authority` ("studies show").
|
|
253
|
+
|
|
254
|
+
## Configure it
|
|
255
|
+
|
|
256
|
+
A TOML config toggles any signal, overrides weights and sets the threshold:
|
|
257
|
+
|
|
258
|
+
```toml
|
|
259
|
+
threshold = 40
|
|
260
|
+
|
|
261
|
+
[signals]
|
|
262
|
+
emoji_density = false # opt a default signal out
|
|
263
|
+
code_undeclared_import = true # opt the import check in
|
|
264
|
+
|
|
265
|
+
[weights]
|
|
266
|
+
ai_self_reference = 6.0
|
|
267
|
+
```
|
|
268
|
+
|
|
269
|
+
Point each surface at it:
|
|
270
|
+
|
|
271
|
+
```sh
|
|
272
|
+
slopscore --config slopscore.toml --files src/*.py # CLI
|
|
273
|
+
git config slopscore.config slopscore.toml # hooks, per repo
|
|
274
|
+
# Action: pass `config: "slopscore.toml"` in the workflow's `with:` block
|
|
275
|
+
```
|
|
276
|
+
|
|
277
|
+
## Honest about its limits
|
|
278
|
+
|
|
279
|
+
- Calibration is validated on the corpus above: humans top out at a score
|
|
280
|
+
of 25, real attributed-AI sits at 70+, and the flag bar (30) lives in the
|
|
281
|
+
empty gap between them. An explicit attribution trailer is a certain tell, so it
|
|
282
|
+
scores HIGH on its own; weak signals must converge to get there.
|
|
283
|
+
- The corpus has an era gap by construction: the human side is pre-2022
|
|
284
|
+
(provably pre-LLM), the AI side is 2023+. Stated, not pretended away - a
|
|
285
|
+
pass against modern human PR-template prose is the known next step.
|
|
286
|
+
- The import check resolves against your manifest; import-name vs package-name
|
|
287
|
+
mismatches (`yaml` vs `PyYAML`) are only partly covered by an alias map -
|
|
288
|
+
hence opt-in. `requirements.txt` `-r` includes are not followed.
|
|
289
|
+
- It is a *signal*, not a judge. A high score means "give this a second read",
|
|
290
|
+
never "this is AI".
|
|
291
|
+
|
|
292
|
+
## Design
|
|
293
|
+
|
|
294
|
+
The detection engine is framework-free; the CLI, git hooks and Action are thin
|
|
295
|
+
front-ends over it. Heuristic-only by design - the value is the discipline
|
|
296
|
+
(low false-positive, evidence-backed, craft not authorship), not a cleverer
|
|
297
|
+
classifier.
|
|
298
|
+
|
|
299
|
+
MIT licensed.
|
|
@@ -0,0 +1,278 @@
|
|
|
1
|
+
# slopscore
|
|
2
|
+
|
|
3
|
+
[](https://github.com/koopatroopa/slopscore/actions/workflows/ci.yml)
|
|
4
|
+
[](https://pypi.org/project/slopscore/)
|
|
5
|
+
[](https://www.python.org/downloads/)
|
|
6
|
+
[](LICENSE)
|
|
7
|
+
|
|
8
|
+
### **_Catch the slop before it ships._**
|
|
9
|
+
|
|
10
|
+
AI tools leave fingerprints — em-dashes, a rocket 🚀 emoji no human would
|
|
11
|
+
ever type by hand, stray `Co-Authored-By: Claude` trailers,
|
|
12
|
+
`# ... rest of the code unchanged` stubs, ghost import declarations,
|
|
13
|
+
`Summary by CodeRabbit` stamps. slopscore detects these AI-generated
|
|
14
|
+
tells by putting a number on them — every commit, push and PR scored out
|
|
15
|
+
of 100, every finding backed by evidence, all before it ships.
|
|
16
|
+
|
|
17
|
+
## Install
|
|
18
|
+
|
|
19
|
+
```sh
|
|
20
|
+
pip install slopscore
|
|
21
|
+
```
|
|
22
|
+
|
|
23
|
+
Then, inside each repo you want watched:
|
|
24
|
+
|
|
25
|
+
```sh
|
|
26
|
+
slopscore install-hooks
|
|
27
|
+
```
|
|
28
|
+
|
|
29
|
+
That second step is the point of the tool - a score on every commit and push.
|
|
30
|
+
Python 3.11 or newer. No other dependencies.
|
|
31
|
+
|
|
32
|
+
Plenty of tools lint AI residue in code. **Nothing else scores the prose that
|
|
33
|
+
ships with it** - the commit messages and PR text where the residue lives in
|
|
34
|
+
git history forever. It starts as a nudge; **one git config turns it into a
|
|
35
|
+
hard gate** that refuses any commit or push over your threshold, and off again
|
|
36
|
+
just as fast.
|
|
37
|
+
|
|
38
|
+
It flags **craft, not authorship**. slopscore never claims "this is AI" - it
|
|
39
|
+
points at fixable leftovers with evidence, like a linter flagging a missing
|
|
40
|
+
semicolon. (Style-based AI detection is unreliable and biased against
|
|
41
|
+
second-language writers, so it refuses to do it: 24 real pre-2020 human PRs
|
|
42
|
+
must score LOW forever - `tests/test_holdout.py`.) Everything runs locally: no
|
|
43
|
+
LLM, no network, no telemetry.
|
|
44
|
+
|
|
45
|
+
## Who runs it
|
|
46
|
+
|
|
47
|
+
Two kinds of people, both on their **own** work:
|
|
48
|
+
|
|
49
|
+
- **You used AI and want to ship clean.** Catch the residue - a leftover
|
|
50
|
+
trailer, a placeholder stub, unedited boilerplate - before a reviewer does.
|
|
51
|
+
Run it on yourself via the hooks, or as a shared standard in CI (report-only
|
|
52
|
+
for repos with outside contributors).
|
|
53
|
+
- **You didn't use AI but fear a detector will say you did.** Formal and
|
|
54
|
+
second-language writers get wrongly flagged constantly. Turn on the thorough
|
|
55
|
+
tier (`--strict`) and slopscore shows your *own* writing through a crude
|
|
56
|
+
detector's eyes - the em-dashes, the "delve", the scaffolding those tools
|
|
57
|
+
latch onto - with evidence, so you can see your exposure and decide whether
|
|
58
|
+
to defang it. slopscore does not believe these prove AI; it refuses that. It
|
|
59
|
+
just lets you see what the flawed tools out there react to.
|
|
60
|
+
|
|
61
|
+
Either way the score is *information*, never an accusation: the verdict is the
|
|
62
|
+
only gate, and honest human work is built to pass.
|
|
63
|
+
|
|
64
|
+
## Tested against reality
|
|
65
|
+
|
|
66
|
+
Every claim above is measured, not asserted. The signals are validated
|
|
67
|
+
against nearly 5,000 real commits and PR bodies from public GitHub history,
|
|
68
|
+
under pre-registered rules: the metric and the pass bar are written down
|
|
69
|
+
before the data is scored, every rejected idea is logged, and a held-out set
|
|
70
|
+
of 24 pre-2020 human PRs is never tuned against - CI enforces that it scores
|
|
71
|
+
LOW forever.
|
|
72
|
+
|
|
73
|
+
- **2,300+ real human commits and PR bodies (pre-2022, definitionally
|
|
74
|
+
pre-LLM): zero false flags.**
|
|
75
|
+
- **1,700+ commits and PR bodies carrying real AI attribution: 99.9%
|
|
76
|
+
flagged.**
|
|
77
|
+
- **785 disciplined, human-reviewed AI-assisted commits: all PASS.** It
|
|
78
|
+
measures residue, not authorship - AI-assisted work that someone actually
|
|
79
|
+
edited scores like human work.
|
|
80
|
+
|
|
81
|
+
When the corpus catches the engine being wrong, the engine changes - and
|
|
82
|
+
signals that false-fired on real humans were rejected and stay rejected.
|
|
83
|
+
|
|
84
|
+
## Try it
|
|
85
|
+
|
|
86
|
+
Scoring a commit on its way out - the message and the staged code, with each
|
|
87
|
+
finding pinned to where it is:
|
|
88
|
+
|
|
89
|
+

|
|
90
|
+
|
|
91
|
+
Or score raw text from the command line:
|
|
92
|
+
|
|
93
|
+
```sh
|
|
94
|
+
echo "Certainly! Let's delve into a robust refactor. Generated with Claude Code." | slopscore --text -
|
|
95
|
+
```
|
|
96
|
+
|
|
97
|
+

|
|
98
|
+
|
|
99
|
+
Drop the "Generated with Claude Code." sentence and it falls to 13.6, PASS -
|
|
100
|
+
single weak signals are normal writing; only convergence flags.
|
|
101
|
+
|
|
102
|
+
The score is a **gradient** - *how much* AI residue is in the text, not a
|
|
103
|
+
yes/no. A low non-zero score is light texture, not an accusation; the
|
|
104
|
+
**verdict** is the only gate, and it is tuned so honest human work passes.
|
|
105
|
+
|
|
106
|
+
- Bands: **LOW** below 30, **MEDIUM** 30 to under 70, **HIGH** 70 and up. The
|
|
107
|
+
**verdict** FLAGs at/above the threshold (default 30), so MEDIUM and HIGH
|
|
108
|
+
both flag.
|
|
109
|
+
- The **exit code** follows the verdict: `0` pass, `1` flag, `2` usage error -
|
|
110
|
+
so it drops straight into CI or a git hook.
|
|
111
|
+
- Evidence carries `path:line` for code; JSON input gets prose locations too
|
|
112
|
+
(`body:14`, `commit[2]:1`).
|
|
113
|
+
|
|
114
|
+
On a terminal the report is coloured by band (green/amber/red); piped output
|
|
115
|
+
stays plain. `--color` and `NO_COLOR` override.
|
|
116
|
+
|
|
117
|
+
## Three ways to use it
|
|
118
|
+
|
|
119
|
+
**1. The CLI - check your work by hand.**
|
|
120
|
+
|
|
121
|
+
```sh
|
|
122
|
+
# score a PR/issue described as JSON ({title, body, commits})
|
|
123
|
+
slopscore pr.json
|
|
124
|
+
# or raw text, or stdin
|
|
125
|
+
echo "Quick fix. Generated with Claude Code." | slopscore --text -
|
|
126
|
+
# scan code files too (go wide on your working tree)
|
|
127
|
+
slopscore --files src/*.py --json
|
|
128
|
+
# thorough tier: also score the opt-in signals (see "Who runs it")
|
|
129
|
+
slopscore --strict pr.json
|
|
130
|
+
```
|
|
131
|
+
|
|
132
|
+
**2. Git hooks - score every commit and push; block them when you say so.**
|
|
133
|
+
|
|
134
|
+
```sh
|
|
135
|
+
slopscore install-hooks
|
|
136
|
+
```
|
|
137
|
+
|
|
138
|
+
Or via the [pre-commit](https://pre-commit.com) framework:
|
|
139
|
+
|
|
140
|
+
```yaml
|
|
141
|
+
repos:
|
|
142
|
+
- repo: https://github.com/koopatroopa/slopscore
|
|
143
|
+
rev: v0.1.0
|
|
144
|
+
hooks:
|
|
145
|
+
- id: slopscore-commit-msg
|
|
146
|
+
- id: slopscore-pre-push
|
|
147
|
+
```
|
|
148
|
+
|
|
149
|
+
`commit-msg` scores your message plus the staged code; `pre-push` scores each
|
|
150
|
+
outgoing commit (flagged ones reported as `[abc1234] Subject`, clean pushes
|
|
151
|
+
silent). **Advisory by default - they never block until you ask.** The gate is
|
|
152
|
+
one setting, per repo, instant in both directions:
|
|
153
|
+
|
|
154
|
+
```sh
|
|
155
|
+
git config slopscore.block true # gate ON: refuse commits/pushes at/above the threshold
|
|
156
|
+
git config slopscore.threshold 50 # move the bar (default 30)
|
|
157
|
+
git config slopscore.block false # gate OFF: back to advisory
|
|
158
|
+
git config slopscore.strict true # thorough tier: score the opt-in signals too
|
|
159
|
+
```
|
|
160
|
+
|
|
161
|
+
Escape hatches even with the gate on: `git commit --no-verify` (or push) skips
|
|
162
|
+
it once; `SLOPSCORE_BLOCK=0` (or `=1`) overrides the setting for one command.
|
|
163
|
+
Every report's footer tells you the current state and the command to flip it.
|
|
164
|
+
|
|
165
|
+
**3. CI - the same gate on every PR, two lines on either platform.**
|
|
166
|
+
|
|
167
|
+
Both recipes score the PR/MR's prose (title + description) plus the diff's
|
|
168
|
+
added lines, report-only until you flip the gate (exit `0` pass, `1` flag).
|
|
169
|
+
It runs on every contributor, so it is a shared craft standard - keep it
|
|
170
|
+
report-only on repos with outside contributors.
|
|
171
|
+
|
|
172
|
+
GitHub (run it under `pull_request`, not `pull_request_target` - slopscore
|
|
173
|
+
only needs to read the diff, never repo write access or secrets):
|
|
174
|
+
|
|
175
|
+
```yaml
|
|
176
|
+
on: pull_request
|
|
177
|
+
permissions:
|
|
178
|
+
contents: read
|
|
179
|
+
# ...
|
|
180
|
+
- uses: actions/checkout@v4
|
|
181
|
+
with: { fetch-depth: 0 }
|
|
182
|
+
- uses: koopatroopa/slopscore@v0 # pin to a tag
|
|
183
|
+
with: { fail-on-flag: "false" } # "true" = gate the merge
|
|
184
|
+
```
|
|
185
|
+
|
|
186
|
+
GitLab:
|
|
187
|
+
|
|
188
|
+
```yaml
|
|
189
|
+
include:
|
|
190
|
+
- remote: https://raw.githubusercontent.com/koopatroopa/slopscore/main/ci/slopscore.gitlab-ci.yml
|
|
191
|
+
```
|
|
192
|
+
|
|
193
|
+
The GitLab job ships advisory (`allow_failure: true`); redeclare the job to
|
|
194
|
+
remove that or set `SLOPSCORE_THRESHOLD`. Any other CI works the same way:
|
|
195
|
+
`pip install slopscore`, feed it `--text` and `--diff`, gate on the exit
|
|
196
|
+
code.
|
|
197
|
+
|
|
198
|
+
## What it looks for
|
|
199
|
+
|
|
200
|
+
Signals that are **on by default** - distinctive leftovers with a very low
|
|
201
|
+
false-positive rate:
|
|
202
|
+
|
|
203
|
+
- `ai_self_reference` - explicit AI attribution: trailers ("Co-Authored-By:
|
|
204
|
+
Claude"), assistant self-talk ("As an AI..."), and the stamps AI review
|
|
205
|
+
bots leave in PR bodies ("## Summary by CodeRabbit", aider's
|
|
206
|
+
auto-generated-PR header)
|
|
207
|
+
- `ai_cliche_phrases` - chatbot filler ("delve into", "it's worth noting")
|
|
208
|
+
- `code_placeholder_stub` - placeholder markers left in code ("// ... rest of
|
|
209
|
+
code", "your implementation here"), reported with file and line.
|
|
210
|
+
- `em_dash_density`, `emoji_density` and `curly_quotes` - U+2014, decorative
|
|
211
|
+
emoji and word-processor quotes are not on your keyboard; in coding
|
|
212
|
+
artefacts they arrive via tooling. These only ever add to a score (they are
|
|
213
|
+
excluded from the normalisation ceiling), so enabling or disabling them
|
|
214
|
+
cannot dilute the signals above - and their combined contribution is
|
|
215
|
+
capped, so however many fire at once they can colour a score but never
|
|
216
|
+
flag on their own. Real Greenkeeper-era PR bodies taught us that one.
|
|
217
|
+
|
|
218
|
+
Signals that are **opt-in**, because humans genuinely type them:
|
|
219
|
+
|
|
220
|
+
- `code_undeclared_import` - an import that is not in the standard library, not
|
|
221
|
+
declared in your manifest and not a local module - possibly a package the
|
|
222
|
+
model made up. It reads your `pyproject`/`requirements` and never imports or
|
|
223
|
+
installs anything.
|
|
224
|
+
- `sycophantic_openers` ("Certainly!", "Hope this helps!") - chat register,
|
|
225
|
+
not commit register: across 2,500+ real AI-attributed commits and PR bodies
|
|
226
|
+
it fired zero times, and its only corpus hits were friendly humans. Kept
|
|
227
|
+
for scanning pasted chat output.
|
|
228
|
+
- `promotional_adjectives` ("robust", "comprehensive"), `section_scaffolding`
|
|
229
|
+
(`## Overview` headers - PR templates generate these), `bold_lead_in_lists`,
|
|
230
|
+
`negative_parallelism` ("not just X, but Y" and its TED-talk cousins),
|
|
231
|
+
`rhetorical_qa` ("Why? Because...") and `vague_authority` ("studies show").
|
|
232
|
+
|
|
233
|
+
## Configure it
|
|
234
|
+
|
|
235
|
+
A TOML config toggles any signal, overrides weights and sets the threshold:
|
|
236
|
+
|
|
237
|
+
```toml
|
|
238
|
+
threshold = 40
|
|
239
|
+
|
|
240
|
+
[signals]
|
|
241
|
+
emoji_density = false # opt a default signal out
|
|
242
|
+
code_undeclared_import = true # opt the import check in
|
|
243
|
+
|
|
244
|
+
[weights]
|
|
245
|
+
ai_self_reference = 6.0
|
|
246
|
+
```
|
|
247
|
+
|
|
248
|
+
Point each surface at it:
|
|
249
|
+
|
|
250
|
+
```sh
|
|
251
|
+
slopscore --config slopscore.toml --files src/*.py # CLI
|
|
252
|
+
git config slopscore.config slopscore.toml # hooks, per repo
|
|
253
|
+
# Action: pass `config: "slopscore.toml"` in the workflow's `with:` block
|
|
254
|
+
```
|
|
255
|
+
|
|
256
|
+
## Honest about its limits
|
|
257
|
+
|
|
258
|
+
- Calibration is validated on the corpus above: humans top out at a score
|
|
259
|
+
of 25, real attributed-AI sits at 70+, and the flag bar (30) lives in the
|
|
260
|
+
empty gap between them. An explicit attribution trailer is a certain tell, so it
|
|
261
|
+
scores HIGH on its own; weak signals must converge to get there.
|
|
262
|
+
- The corpus has an era gap by construction: the human side is pre-2022
|
|
263
|
+
(provably pre-LLM), the AI side is 2023+. Stated, not pretended away - a
|
|
264
|
+
pass against modern human PR-template prose is the known next step.
|
|
265
|
+
- The import check resolves against your manifest; import-name vs package-name
|
|
266
|
+
mismatches (`yaml` vs `PyYAML`) are only partly covered by an alias map -
|
|
267
|
+
hence opt-in. `requirements.txt` `-r` includes are not followed.
|
|
268
|
+
- It is a *signal*, not a judge. A high score means "give this a second read",
|
|
269
|
+
never "this is AI".
|
|
270
|
+
|
|
271
|
+
## Design
|
|
272
|
+
|
|
273
|
+
The detection engine is framework-free; the CLI, git hooks and Action are thin
|
|
274
|
+
front-ends over it. Heuristic-only by design - the value is the discipline
|
|
275
|
+
(low false-positive, evidence-backed, craft not authorship), not a cleverer
|
|
276
|
+
classifier.
|
|
277
|
+
|
|
278
|
+
MIT licensed.
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=61"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "slopscore"
|
|
7
|
+
version = "0.1.0"
|
|
8
|
+
description = "Heuristic linter that scores AI residue in commits and PRs 0-100 with evidence - flags craft, not authorship"
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
license = { text = "MIT" }
|
|
11
|
+
requires-python = ">=3.11"
|
|
12
|
+
authors = [{ name = "Matt Whelan" }]
|
|
13
|
+
keywords = ["lint", "ai", "slop", "commit", "git-hooks", "pull-request", "code-quality", "ci"]
|
|
14
|
+
classifiers = [
|
|
15
|
+
"Development Status :: 4 - Beta",
|
|
16
|
+
"Environment :: Console",
|
|
17
|
+
"Intended Audience :: Developers",
|
|
18
|
+
"Operating System :: OS Independent",
|
|
19
|
+
"Programming Language :: Python :: 3",
|
|
20
|
+
"Topic :: Software Development :: Quality Assurance",
|
|
21
|
+
]
|
|
22
|
+
dependencies = []
|
|
23
|
+
|
|
24
|
+
[project.urls]
|
|
25
|
+
Homepage = "https://github.com/koopatroopa/slopscore"
|
|
26
|
+
Repository = "https://github.com/koopatroopa/slopscore"
|
|
27
|
+
Issues = "https://github.com/koopatroopa/slopscore/issues"
|
|
28
|
+
|
|
29
|
+
[project.scripts]
|
|
30
|
+
slopscore = "slopscore.cli:main"
|
|
31
|
+
|
|
32
|
+
[tool.setuptools]
|
|
33
|
+
package-dir = {"" = "src"}
|
|
34
|
+
packages = ["slopscore"]
|
|
35
|
+
|
|
36
|
+
[tool.pytest.ini_options]
|
|
37
|
+
testpaths = ["tests"]
|
|
38
|
+
pythonpath = ["."]
|
|
39
|
+
|
|
40
|
+
[dependency-groups]
|
|
41
|
+
dev = ["pytest"]
|