zero-slop 2.12.29 → 2.12.31
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -3
- package/SKILL.md +17 -1
- package/bin/lib/text-hygiene.d.mts +6 -0
- package/bin/lib/text-hygiene.mjs +111 -0
- package/bin/zero-slop.mjs +13 -2
- package/docs/cli.md +17 -1
- package/docs/rest-api.md +2 -1
- package/gemini-extension.json +1 -1
- package/package.json +1 -1
- package/plugin.json +1 -1
- package/scripts/text_hygiene.py +143 -0
package/README.md
CHANGED
|
@@ -14,7 +14,7 @@ It finds AI slop and workslop, guides the edit, and checks that the key details
|
|
|
14
14
|
|
|
15
15
|
<p align="center">
|
|
16
16
|
<a href="https://github.com/manavmishra/ZeroSlop/actions/workflows/validate.yml"><img alt="Validate" src="https://github.com/manavmishra/ZeroSlop/actions/workflows/validate.yml/badge.svg"></a>
|
|
17
|
-
<img alt="Version 2.12.
|
|
17
|
+
<img alt="Version 2.12.31" src="https://img.shields.io/badge/version-2.12.31-72528F?color=C15732">
|
|
18
18
|
<a href="https://www.npmjs.com/package/zero-slop"><img alt="npm version" src="https://img.shields.io/npm/v/zero-slop?color=C15732"></a>
|
|
19
19
|
<a href="https://www.npmjs.com/package/zero-slop"><img alt="npm downloads" src="https://img.shields.io/npm/dm/zero-slop?color=17634F"></a>
|
|
20
20
|
<a href="https://github.com/manavmishra/ZeroSlop/stargazers"><img alt="GitHub stars" src="https://img.shields.io/github/stars/manavmishra/ZeroSlop?style=flat&color=C15732"></a>
|
|
@@ -120,7 +120,7 @@ We ran Zero Slop and three other open-source agent skills on our AI Slop test co
|
|
|
120
120
|
|
|
121
121
|

|
|
122
122
|
|
|
123
|
-
|
|
123
|
+
Our [RAID+ audit](bench/raid-plus-corpus/README.md) scored 7,627 non-empty machine-generated abstracts from four models. The [INSCONE project](https://github.com/markstanl/INSCONE) extended RAID by generating new outputs from the original RAID prompts.
|
|
124
124
|
|
|
125
125
|
| Model | Texts scored | Mean writing score ↓ | At or above 25 |
|
|
126
126
|
|---|---:|---:|---:|
|
|
@@ -176,7 +176,7 @@ python3 scripts/slopscore.py --batch drafts/ --gate 25
|
|
|
176
176
|
The CLI sends a file to the hosted editor without changing the file on disk:
|
|
177
177
|
|
|
178
178
|
```sh
|
|
179
|
-
npx --yes zero-slop@2.12.
|
|
179
|
+
npx --yes zero-slop@2.12.31 deslop draft.md --genre professional
|
|
180
180
|
```
|
|
181
181
|
|
|
182
182
|
Use `-` for stdin and `--json` for structured output. `--require-approved` prints the result but exits nonzero when review is needed. Requires Node.js 22+; offline `score` also needs Python 3. [CLI options and privacy](docs/cli.md).
|
package/SKILL.md
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
name: zero-slop
|
|
3
3
|
license: MIT
|
|
4
4
|
metadata:
|
|
5
|
-
version: "2.12.
|
|
5
|
+
version: "2.12.31"
|
|
6
6
|
author: manavmishra
|
|
7
7
|
description: Edit drafts into natural prose, inspect AI-sounding patterns, or review how a specified audience might respond passage by passage. Zero Slop runs inside the user's existing AI assistant with local tools that protect source details. Use for humanizing or de-slopping writing, polishing outward-facing prose, social drafts, final editorial checks, or an explicit simulated reader review. Preserve facts, voice and format; reader simulations are hypotheses, not human feedback.
|
|
8
8
|
---
|
|
@@ -33,6 +33,22 @@ citations, and the ladder below orders the signals by measured strength.
|
|
|
33
33
|
|
|
34
34
|
## Hard rules (non-negotiable)
|
|
35
35
|
|
|
36
|
+
For an explicit character-cleanup request, use the bundled offline text-hygiene
|
|
37
|
+
helper at `scripts/text_hygiene.py` when available. It inspects formatting
|
|
38
|
+
characters and removes only a
|
|
39
|
+
small reviewed set of embedded artifacts in 16 common ASCII prose words.
|
|
40
|
+
Only sentence-shaped lines with conservative prose openers are eligible.
|
|
41
|
+
Recognized code, command syntax, URLs, formulas, quotations and multilingual
|
|
42
|
+
formatting are excluded; these guards are not a complete code parser. Preserve
|
|
43
|
+
uncertain text and use inspection when its purpose is unclear.
|
|
44
|
+
This is a separate local operation, not a replacement for contextual editing.
|
|
45
|
+
Its change record can reverse the exact cleanup. A scriptless assistant must
|
|
46
|
+
not claim byte-level cleanup it did not perform. Character cleanup does not
|
|
47
|
+
verify removal of statistical watermarks from ChatGPT, Claude, Gemini or any
|
|
48
|
+
other model. Preserve required disclosures and attribution.
|
|
49
|
+
In a host that routes local checks through the bundled MCP connector, use its
|
|
50
|
+
`clean_text` tool with the supplied text and optional `inspect: true` instead.
|
|
51
|
+
|
|
36
52
|
1. **Fidelity.** Meaning, claims, and facts survive exactly. Never invent a
|
|
37
53
|
number, name, anecdote, or experience — and experiential/interior claims
|
|
38
54
|
count ("by test day it felt familiar", "I was terrified"): if the author
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
export type HygieneEdit = { start: number; end: number; original: string; replacement: ''; reason: string; outputStart: number; outputBinding: string };
|
|
2
|
+
export type HygieneResult = { text: string; edits: HygieneEdit[]; status: 'cleaned' | 'unchanged'; policyVersion: string; offsetUnit: 'utf16'; statisticalWatermark: { status: 'not_verified'; reason: string } };
|
|
3
|
+
export function cleanTextHygiene(text: string): HygieneResult;
|
|
4
|
+
export function inspectTextHygiene(text: string): Omit<HygieneResult, 'status'> & { status: 'inspected'; findings: Array<{ start: number; codePoint: string; disposition: 'removable' | 'preserved' }> };
|
|
5
|
+
export function reverseTextHygiene(text: string, edits: HygieneEdit[]): string;
|
|
6
|
+
export function createTextHygiene(): { cleanTextHygiene: typeof cleanTextHygiene; inspectTextHygiene: typeof inspectTextHygiene; reverseTextHygiene: typeof reverseTextHygiene };
|
|
@@ -0,0 +1,111 @@
|
|
|
1
|
+
/** Conservative copy-artifact cleanup. This never detects a provider watermark. */
|
|
2
|
+
export function createTextHygiene() {
|
|
3
|
+
const policyVersion = '2026-10-26.1';
|
|
4
|
+
const removable = /[\u200B\u2060\uFEFF]/g;
|
|
5
|
+
const words = new Set('a an and are clear draft hello in is of text the this to world writing'.split(' '));
|
|
6
|
+
const statisticalWatermark = { status: 'not_verified', reason: 'matching_detector_unavailable' };
|
|
7
|
+
function validate(text) {
|
|
8
|
+
if (typeof text !== 'string') throw new TypeError('Text must be a string.');
|
|
9
|
+
if (text.length > 200000) throw new RangeError('Text hygiene input exceeds 200000 UTF-16 units.');
|
|
10
|
+
}
|
|
11
|
+
// A change-detection fingerprint, not authentication or provenance evidence.
|
|
12
|
+
function binding(text) {
|
|
13
|
+
let first = 2166136261; let second = 2246822519;
|
|
14
|
+
for (let i = 0; i < text.length; i += 1) {
|
|
15
|
+
first = Math.imul(first ^ text.charCodeAt(i), 16777619) >>> 0;
|
|
16
|
+
second = Math.imul(second ^ text.charCodeAt(i), 3266489917) >>> 0;
|
|
17
|
+
}
|
|
18
|
+
return `${text.length}:${first}:${second}`;
|
|
19
|
+
}
|
|
20
|
+
function plannedEdits(text) {
|
|
21
|
+
if (/[<>]/.test(text)) return []; // Preserve unknown markup and multiline HTML as a whole.
|
|
22
|
+
const edits = []; let offset = 0; let fence = null; let frontmatter = false;
|
|
23
|
+
for (const line of text.split(/(?<=\n)/)) {
|
|
24
|
+
if (offset === 0 && /^\uFEFF?---\s*$/.test(line)) { frontmatter = true; offset += line.length; continue; }
|
|
25
|
+
if (frontmatter) { if (/^(?:---|\.\.\.)\s*$/.test(line)) frontmatter = false; offset += line.length; continue; }
|
|
26
|
+
const marker = line.match(/^\s*(`{3,}|~{3,})/);
|
|
27
|
+
if (marker) {
|
|
28
|
+
if (!fence) fence = { char: marker[1][0], length: marker[1].length };
|
|
29
|
+
else if (fence.char === marker[1][0] && marker[1].length >= fence.length && /^\s*(?:`+|~+)\s*$/.test(line)) fence = null;
|
|
30
|
+
offset += line.length; continue;
|
|
31
|
+
}
|
|
32
|
+
const plain = line.replace(removable, '');
|
|
33
|
+
// Sentence shape and a small set of prose openers keep unfenced commands
|
|
34
|
+
// and unknown fragments out of this deliberately conservative cleanup.
|
|
35
|
+
const protectedLine = fence || !/^\s*(?:the|this|hello|an?)\b/i.test(plain) || !/[.!?]\s*$/.test(plain)
|
|
36
|
+
|| /[()[\]{};&]|(?:^|\s)--?[A-Za-z]/.test(plain)
|
|
37
|
+
|| [...plain].some(char => (char.charCodeAt(0) < 32 && !'\t\n\r'.includes(char)) || char.charCodeAt(0) > 126)
|
|
38
|
+
|| /^ {4}|^\t|[`"'<>$=\\:|]|https?:|www\.|@|\]\(|\b(?:identifier|variable|function|path|command|token|password|code|regex|formula|equation)\b/i.test(plain)
|
|
39
|
+
|| /\b\w+[./_]\w+\b/.test(plain);
|
|
40
|
+
const ordinaryWords = plain.match(/[A-Za-z]+/g) || [];
|
|
41
|
+
if (!protectedLine && ordinaryWords.length >= 2) {
|
|
42
|
+
for (const word of line.matchAll(/[A-Za-z\u200B\u2060\uFEFF]+/g)) {
|
|
43
|
+
const cleaned = word[0].replace(removable, '');
|
|
44
|
+
if (!words.has(cleaned.toLowerCase())) continue;
|
|
45
|
+
for (let i = 1; i < word[0].length - 1; i += 1) {
|
|
46
|
+
const char = word[0][i];
|
|
47
|
+
if ('\u200B\u2060\uFEFF'.includes(char) && /[A-Za-z]/.test(word[0][i - 1]) && /[A-Za-z]/.test(word[0][i + 1])) {
|
|
48
|
+
const start = offset + word.index + i;
|
|
49
|
+
edits.push({ start, end: start + 1, original: char, replacement: '', reason: 'embedded_ascii_copy_artifact' });
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
offset += line.length;
|
|
55
|
+
}
|
|
56
|
+
return edits;
|
|
57
|
+
}
|
|
58
|
+
function cleanTextHygiene(text) {
|
|
59
|
+
validate(text);
|
|
60
|
+
const edits = plannedEdits(text);
|
|
61
|
+
const chunks = []; let cursor = 0;
|
|
62
|
+
for (const edit of edits) { chunks.push(text.slice(cursor, edit.start)); cursor = edit.end; }
|
|
63
|
+
chunks.push(text.slice(cursor));
|
|
64
|
+
const cleaned = chunks.join(''); const outputBinding = binding(cleaned);
|
|
65
|
+
for (let i = 0; i < edits.length; i += 1) {
|
|
66
|
+
edits[i].outputStart = edits[i].start - i;
|
|
67
|
+
edits[i].outputBinding = outputBinding;
|
|
68
|
+
}
|
|
69
|
+
return { text: cleaned, edits, status: edits.length ? 'cleaned' : 'unchanged', policyVersion,
|
|
70
|
+
offsetUnit: 'utf16', statisticalWatermark: { ...statisticalWatermark } };
|
|
71
|
+
}
|
|
72
|
+
function inspectTextHygiene(text) {
|
|
73
|
+
const result = cleanTextHygiene(text); const findings = [];
|
|
74
|
+
const actionable = new Set(result.edits.map(edit => edit.start));
|
|
75
|
+
for (let i = 0; i < text.length;) {
|
|
76
|
+
const cp = text.codePointAt(i);
|
|
77
|
+
if (cp === 0xAD || cp === 0xFEFF || (cp >= 0x200B && cp <= 0x200F)
|
|
78
|
+
|| (cp >= 0x202A && cp <= 0x202E) || (cp >= 0x2060 && cp <= 0x2069)
|
|
79
|
+
|| (cp >= 0xFE00 && cp <= 0xFE0F) || (cp >= 0xE0000 && cp <= 0xE01EF)) {
|
|
80
|
+
findings.push({ start: i, codePoint: `U+${cp.toString(16).toUpperCase().padStart(4, '0')}`,
|
|
81
|
+
disposition: actionable.has(i) ? 'removable' : 'preserved' });
|
|
82
|
+
}
|
|
83
|
+
i += cp > 0xFFFF ? 2 : 1;
|
|
84
|
+
}
|
|
85
|
+
return { ...result, text, findings, status: 'inspected' };
|
|
86
|
+
}
|
|
87
|
+
function reverseTextHygiene(text, edits) {
|
|
88
|
+
validate(text);
|
|
89
|
+
if (!Array.isArray(edits) || edits.length > 200000) throw new TypeError('Edits must be a bounded array.');
|
|
90
|
+
const chunks = []; let cursor = 0; let previous = -1; const outputBinding = binding(text);
|
|
91
|
+
for (let i = 0; i < edits.length; i += 1) {
|
|
92
|
+
const edit = edits[i];
|
|
93
|
+
if (!edit || typeof edit !== 'object' || !Number.isInteger(edit.start) || edit.start < 1 || edit.start > text.length + edits.length
|
|
94
|
+
|| !Number.isInteger(edit.outputStart) || edit.outputStart < 0 || edit.outputStart > text.length
|
|
95
|
+
|| edit.start <= previous || edit.end !== edit.start + 1
|
|
96
|
+
|| !'\u200B\u2060\uFEFF'.includes(edit.original) || edit.original.length !== 1
|
|
97
|
+
|| edit.replacement !== '' || edit.outputBinding !== outputBinding
|
|
98
|
+
|| edit.outputStart !== edit.start - i) throw new Error('Cleanup plan does not match this text.');
|
|
99
|
+
previous = edit.start;
|
|
100
|
+
chunks.push(text.slice(cursor, edit.outputStart), edit.original); cursor = edit.outputStart;
|
|
101
|
+
}
|
|
102
|
+
chunks.push(text.slice(cursor));
|
|
103
|
+
return chunks.join('');
|
|
104
|
+
}
|
|
105
|
+
return Object.freeze({ cleanTextHygiene, inspectTextHygiene, reverseTextHygiene });
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
const helpers = createTextHygiene();
|
|
109
|
+
export const cleanTextHygiene = helpers.cleanTextHygiene;
|
|
110
|
+
export const inspectTextHygiene = helpers.inspectTextHygiene;
|
|
111
|
+
export const reverseTextHygiene = helpers.reverseTextHygiene;
|
package/bin/zero-slop.mjs
CHANGED
|
@@ -12,6 +12,7 @@ import { homedir } from "node:os";
|
|
|
12
12
|
import { dirname, join, parse, resolve } from "node:path";
|
|
13
13
|
import { fileURLToPath } from "node:url";
|
|
14
14
|
import { deslop, DeslopError, isApprovedResult, validateInput } from "./lib/deslop.mjs";
|
|
15
|
+
import { cleanTextHygiene, inspectTextHygiene } from "./lib/text-hygiene.mjs";
|
|
15
16
|
|
|
16
17
|
const ROOT = resolve(dirname(fileURLToPath(import.meta.url)), "..");
|
|
17
18
|
const PAYLOAD = ["SKILL.md", "references", "scripts", "data"];
|
|
@@ -37,6 +38,7 @@ function usage(v) {
|
|
|
37
38
|
Usage
|
|
38
39
|
npx zero-slop install [--harness <name>] [--dir <path>] [--force]
|
|
39
40
|
npx zero-slop score <file|-> [-- <slopscore flags>]
|
|
41
|
+
npx zero-slop clean-text <file|-> [--inspect] [--json]
|
|
40
42
|
npx zero-slop deslop <file|-> [--genre <name>] [--audience <reader>] [--json]
|
|
41
43
|
[--require-approved] [--timeout <seconds>]
|
|
42
44
|
npx zero-slop where
|
|
@@ -249,7 +251,7 @@ function parseDeslopArgs(argv) {
|
|
|
249
251
|
json: Boolean(values["--json"]), requireApproved: Boolean(values["--require-approved"]), timeoutMs };
|
|
250
252
|
}
|
|
251
253
|
|
|
252
|
-
async function readDraft(file, signal) {
|
|
254
|
+
async function readDraft(file, signal, preserveBOM = false) {
|
|
253
255
|
const maximum = 128 * 1024;
|
|
254
256
|
let bytes;
|
|
255
257
|
if (file === "-") {
|
|
@@ -288,7 +290,7 @@ async function readDraft(file, signal) {
|
|
|
288
290
|
} finally { await handle?.close(); }
|
|
289
291
|
}
|
|
290
292
|
signal.throwIfAborted();
|
|
291
|
-
try { return new TextDecoder("utf-8", { fatal: true }).decode(bytes); } catch {
|
|
293
|
+
try { return new TextDecoder("utf-8", { fatal: true, ignoreBOM: preserveBOM }).decode(bytes); } catch {
|
|
292
294
|
throw new DeslopError("invalid_input", "The draft must be valid UTF-8; nothing was uploaded.");
|
|
293
295
|
}
|
|
294
296
|
}
|
|
@@ -348,6 +350,15 @@ async function runDeslop(argv, v) {
|
|
|
348
350
|
|
|
349
351
|
async function main() {
|
|
350
352
|
const argv = process.argv.slice(2);
|
|
353
|
+
if (argv[0] === "clean-text") {
|
|
354
|
+
const args = argv.slice(1);
|
|
355
|
+
const files = args.filter(arg => !["--inspect", "--json"].includes(arg));
|
|
356
|
+
if (files.length !== 1 || (files[0].startsWith("--"))) throw new Error("clean-text needs one file or -, plus optional --inspect and --json");
|
|
357
|
+
const text = await readDraft(files[0], new AbortController().signal, true);
|
|
358
|
+
const result = args.includes("--inspect") ? inspectTextHygiene(text) : cleanTextHygiene(text);
|
|
359
|
+
process.stdout.write(args.includes("--json") ? `${JSON.stringify(result)}\n` : result.text);
|
|
360
|
+
return 0;
|
|
361
|
+
}
|
|
351
362
|
// Parse this command separately: no install flags or scorer passthrough can
|
|
352
363
|
// accidentally become a remote-upload option.
|
|
353
364
|
if (argv[0] === "deslop") {
|
package/docs/cli.md
CHANGED
|
@@ -1,9 +1,25 @@
|
|
|
1
1
|
# Zero Slop CLI
|
|
2
2
|
|
|
3
|
+
For an explicit local copy-artifact cleanup, use `npx zero-slop clean-text draft.md`.
|
|
4
|
+
Use `--inspect --json` to inspect without changing the returned text, or `--json`
|
|
5
|
+
with cleanup to receive reversible edits. The command never changes the source
|
|
6
|
+
file and accepts one regular UTF-8 file or stdin, up to 128 KiB.
|
|
7
|
+
|
|
8
|
+
This first policy removes only embedded U+200B, U+2060 and U+FEFF in a small
|
|
9
|
+
reviewed vocabulary of 16 common ASCII prose words. A line is eligible only
|
|
10
|
+
when it starts with The, This, Hello, A or An and ends with sentence punctuation.
|
|
11
|
+
Recognized code, command flags, operators, quotes, URLs, markup and non-ASCII
|
|
12
|
+
text are excluded. Other words and fragments stay unchanged. These guards are
|
|
13
|
+
conservative rules, not a complete parser for every programming language.
|
|
14
|
+
This deliberately limited cleanup does not verify statistical watermark removal.
|
|
15
|
+
The installed skill also includes the offline Python helper. Its offsets use
|
|
16
|
+
code points; the JavaScript result uses UTF-16 offsets. Change-detection bindings
|
|
17
|
+
are not authentication or provenance evidence.
|
|
18
|
+
|
|
3
19
|
Use `deslop` for hosted editing with MCP parity. Use `score` for offline checks.
|
|
4
20
|
|
|
5
21
|
```sh
|
|
6
|
-
npm install --global zero-slop@2.12.
|
|
22
|
+
npm install --global zero-slop@2.12.31
|
|
7
23
|
zero-slop deslop draft.md --genre professional
|
|
8
24
|
zero-slop deslop - --genre email < draft.txt
|
|
9
25
|
zero-slop deslop draft.md --json --require-approved
|
package/docs/rest-api.md
CHANGED
|
@@ -22,9 +22,10 @@ curl --fail-with-body --max-time 75 --include https://mcp.zero-slop.ai/v1/deslop
|
|
|
22
22
|
|
|
23
23
|
| Field | Accepted value |
|
|
24
24
|
|---|---|
|
|
25
|
-
| `text` | Required, nonempty string; up to 20,000 Unicode code points
|
|
25
|
+
| `text` | Required, nonempty string; up to 20,000 Unicode code points. Editing trims the text first; `clean_text` preserves it and counts its full length. |
|
|
26
26
|
| `genre` | `general` (default), `social`, `email`, `research`, or `professional` |
|
|
27
27
|
| `audience` | Optional string, up to 200 Unicode code points after trimming |
|
|
28
|
+
| `operation` | Optional `"clean_text"` for conservative character cleanup without an editing-model call |
|
|
28
29
|
|
|
29
30
|
Lengths count Unicode code points; a combined emoji or accented character can contain more than one.
|
|
30
31
|
Send UTF-8 JSON in the body, never a URL query. The body limit is 128 KiB, including
|
package/gemini-extension.json
CHANGED
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "zero-slop",
|
|
3
|
-
"version": "2.12.
|
|
3
|
+
"version": "2.12.31",
|
|
4
4
|
"description": "An editing skill, offline writing checks, and a CLI for the hosted Zero Slop MCP pipeline. Find stock AI phrasing, rewrite, and check source details.",
|
|
5
5
|
"bin": {
|
|
6
6
|
"zero-slop": "bin/zero-slop.mjs"
|
package/plugin.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"$schema": "https://agent-plugins.org/schemas/1.0.0/plugin.schema.json",
|
|
3
3
|
"name": "zero-slop",
|
|
4
|
-
"version": "2.12.
|
|
4
|
+
"version": "2.12.31",
|
|
5
5
|
"description": "Find and fix canned AI writing while preserving the facts, voice, and format.",
|
|
6
6
|
"author": {
|
|
7
7
|
"name": "Manav Mishra",
|
|
@@ -0,0 +1,143 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""Offline, conservative copy-artifact cleanup; not a watermark detector."""
|
|
3
|
+
import argparse
|
|
4
|
+
import hashlib
|
|
5
|
+
import json
|
|
6
|
+
import os
|
|
7
|
+
import re
|
|
8
|
+
import stat
|
|
9
|
+
import sys
|
|
10
|
+
|
|
11
|
+
POLICY_VERSION = "2026-10-26.1"
|
|
12
|
+
REMOVABLE = re.compile("[\u200b\u2060\ufeff]")
|
|
13
|
+
WORDS = frozenset("a an and are clear draft hello in is of text the this to world writing".split())
|
|
14
|
+
PROTECTED = re.compile(r'''^ {4}|^\t|[`"'<>$=\\:|]|https?:|www\.|@|\]\(|\b(?:identifier|variable|function|path|command|token|password|code|regex|formula|equation)\b|\b\w+[./_]\w+\b''', re.I)
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def _validate(text):
|
|
18
|
+
if not isinstance(text, str):
|
|
19
|
+
raise TypeError("Text must be a string.")
|
|
20
|
+
if len(text.encode("utf-16-le", errors="surrogatepass")) // 2 > 200000:
|
|
21
|
+
raise ValueError("Text hygiene input exceeds 200000 UTF-16 units.")
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def clean_text_hygiene(text):
|
|
25
|
+
_validate(text)
|
|
26
|
+
edits, offset, fence, frontmatter = [], 0, None, False
|
|
27
|
+
markup = "<" in text or ">" in text
|
|
28
|
+
parts = text.split("\n")
|
|
29
|
+
for line_index, part in enumerate(parts):
|
|
30
|
+
line = part + ("\n" if line_index < len(parts) - 1 else "")
|
|
31
|
+
if offset == 0 and re.fullmatch("\ufeff?---\\s*", line):
|
|
32
|
+
frontmatter = True
|
|
33
|
+
offset += len(line)
|
|
34
|
+
continue
|
|
35
|
+
if frontmatter:
|
|
36
|
+
if re.fullmatch(r"(?:---|\.\.\.)\s*", line):
|
|
37
|
+
frontmatter = False
|
|
38
|
+
offset += len(line)
|
|
39
|
+
continue
|
|
40
|
+
marker = re.match(r"\s*(`{3,}|~{3,})", line)
|
|
41
|
+
if marker:
|
|
42
|
+
char = marker[1][0]
|
|
43
|
+
if not fence:
|
|
44
|
+
fence = (char, len(marker[1]))
|
|
45
|
+
elif fence[0] == char and len(marker[1]) >= fence[1] and re.fullmatch(r"\s*(?:`+|~+)\s*", line):
|
|
46
|
+
fence = None
|
|
47
|
+
offset += len(line)
|
|
48
|
+
continue
|
|
49
|
+
plain = REMOVABLE.sub("", line)
|
|
50
|
+
# Preserve unfenced commands and fragments: only conservative prose
|
|
51
|
+
# openers with sentence-ending punctuation are eligible for cleanup.
|
|
52
|
+
protected = (markup or fence or not re.match(r"\s*(?:the|this|hello|an?)\b", plain, re.I)
|
|
53
|
+
or not re.search(r"[.!?]\s*$", plain)
|
|
54
|
+
or re.search(r"[()[\]{};&]|(?:^|\s)--?[A-Za-z]", plain)
|
|
55
|
+
or re.search(r"[^\x09\x0a\x0d\x20-\x7e]", plain) or PROTECTED.search(plain))
|
|
56
|
+
if not protected and len(re.findall(r"[A-Za-z]+", plain)) >= 2:
|
|
57
|
+
for match in re.finditer("[A-Za-z\u200b\u2060\ufeff]+", line):
|
|
58
|
+
word = match[0]
|
|
59
|
+
if REMOVABLE.sub("", word).lower() not in WORDS:
|
|
60
|
+
continue
|
|
61
|
+
for index in range(1, len(word) - 1):
|
|
62
|
+
if word[index] in "\u200b\u2060\ufeff" and re.fullmatch("[A-Za-z]", word[index - 1]) and re.fullmatch("[A-Za-z]", word[index + 1]):
|
|
63
|
+
start = offset + match.start() + index
|
|
64
|
+
edits.append({"start": start, "end": start + 1, "original": word[index], "replacement": "", "reason": "embedded_ascii_copy_artifact"})
|
|
65
|
+
offset += len(line)
|
|
66
|
+
chunks, cursor = [], 0
|
|
67
|
+
for edit in edits:
|
|
68
|
+
chunks.append(text[cursor:edit["start"]])
|
|
69
|
+
cursor = edit["end"]
|
|
70
|
+
chunks.append(text[cursor:])
|
|
71
|
+
cleaned = "".join(chunks)
|
|
72
|
+
digest = hashlib.sha256(cleaned.encode("utf-8", errors="surrogatepass")).hexdigest()
|
|
73
|
+
for index, edit in enumerate(edits):
|
|
74
|
+
edit.update(outputStart=edit["start"] - index, outputBinding=digest)
|
|
75
|
+
return {"text": cleaned, "edits": edits, "status": "cleaned" if edits else "unchanged", "policyVersion": POLICY_VERSION,
|
|
76
|
+
"offsetUnit": "codepoint", "statisticalWatermark": {"status": "not_verified", "reason": "matching_detector_unavailable"}}
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def inspect_text_hygiene(text):
|
|
80
|
+
result = clean_text_hygiene(text)
|
|
81
|
+
actionable = {edit["start"] for edit in result["edits"]}
|
|
82
|
+
findings = []
|
|
83
|
+
for index, char in enumerate(text):
|
|
84
|
+
cp = ord(char)
|
|
85
|
+
if cp in (0xad, 0xfeff) or 0x200b <= cp <= 0x200f or 0x202a <= cp <= 0x202e or 0x2060 <= cp <= 0x2069 or 0xfe00 <= cp <= 0xfe0f or 0xe0000 <= cp <= 0xe01ef:
|
|
86
|
+
findings.append({"start": index, "codePoint": f"U+{cp:04X}", "disposition": "removable" if index in actionable else "preserved"})
|
|
87
|
+
return dict(result, text=text, findings=findings, status="inspected")
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def reverse_text_hygiene(text, edits):
|
|
91
|
+
_validate(text)
|
|
92
|
+
if not isinstance(edits, list) or len(edits) > 200000:
|
|
93
|
+
raise ValueError("Edits must be a bounded array.")
|
|
94
|
+
digest = hashlib.sha256(text.encode("utf-8", errors="surrogatepass")).hexdigest()
|
|
95
|
+
chunks, cursor, previous = [], 0, -1
|
|
96
|
+
for index, edit in enumerate(edits):
|
|
97
|
+
if not isinstance(edit, dict):
|
|
98
|
+
raise ValueError("Cleanup plan does not match this text.")
|
|
99
|
+
start = edit.get("start")
|
|
100
|
+
output_start = edit.get("outputStart")
|
|
101
|
+
if (type(start) is not int or not 1 <= start <= len(text) + len(edits)
|
|
102
|
+
or type(output_start) is not int or not 0 <= output_start <= len(text)
|
|
103
|
+
or start <= previous or type(edit.get("end")) is not int
|
|
104
|
+
or edit["end"] != start + 1
|
|
105
|
+
or edit.get("original") not in ("\u200b", "\u2060", "\ufeff")
|
|
106
|
+
or edit.get("replacement") != "" or edit.get("outputBinding") != digest
|
|
107
|
+
or output_start != start - index):
|
|
108
|
+
raise ValueError("Cleanup plan does not match this text.")
|
|
109
|
+
previous = start
|
|
110
|
+
chunks.extend((text[cursor:output_start], edit["original"]))
|
|
111
|
+
cursor = output_start
|
|
112
|
+
chunks.append(text[cursor:])
|
|
113
|
+
return "".join(chunks)
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
def main():
|
|
117
|
+
parser = argparse.ArgumentParser(description=__doc__)
|
|
118
|
+
parser.add_argument("path", help="UTF-8 file or - for stdin")
|
|
119
|
+
parser.add_argument("--inspect", action="store_true")
|
|
120
|
+
parser.add_argument("--json", action="store_true")
|
|
121
|
+
args = parser.parse_args()
|
|
122
|
+
maximum = 128 * 1024
|
|
123
|
+
if args.path == "-":
|
|
124
|
+
payload = sys.stdin.buffer.read(maximum + 1)
|
|
125
|
+
else:
|
|
126
|
+
descriptor = os.open(args.path, os.O_RDONLY | getattr(os, "O_NONBLOCK", 0))
|
|
127
|
+
with os.fdopen(descriptor, "rb") as source:
|
|
128
|
+
if not stat.S_ISREG(os.fstat(source.fileno()).st_mode):
|
|
129
|
+
raise ValueError("Provide a regular UTF-8 file.")
|
|
130
|
+
payload = source.read(maximum + 1)
|
|
131
|
+
if len(payload) > maximum:
|
|
132
|
+
raise ValueError("Input exceeds 128 KiB.")
|
|
133
|
+
text = payload.decode("utf-8", errors="strict")
|
|
134
|
+
result = inspect_text_hygiene(text) if args.inspect else clean_text_hygiene(text)
|
|
135
|
+
print(json.dumps(result, ensure_ascii=True) if args.json else result["text"], end="\n" if args.json else "")
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
if __name__ == "__main__":
|
|
139
|
+
try:
|
|
140
|
+
main()
|
|
141
|
+
except (OSError, ValueError, TypeError):
|
|
142
|
+
print("Text cleanup failed: provide bounded, valid UTF-8 input.", file=sys.stderr)
|
|
143
|
+
raise SystemExit(2)
|