zero-slop 2.12.29 → 2.12.31

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -14,7 +14,7 @@ It finds AI slop and workslop, guides the edit, and checks that the key details
14
14
 
15
15
  <p align="center">
16
16
  <a href="https://github.com/manavmishra/ZeroSlop/actions/workflows/validate.yml"><img alt="Validate" src="https://github.com/manavmishra/ZeroSlop/actions/workflows/validate.yml/badge.svg"></a>
17
- <img alt="Version 2.12.29" src="https://img.shields.io/badge/version-2.12.29-72528F?color=C15732">
17
+ <img alt="Version 2.12.31" src="https://img.shields.io/badge/version-2.12.31-72528F?color=C15732">
18
18
  <a href="https://www.npmjs.com/package/zero-slop"><img alt="npm version" src="https://img.shields.io/npm/v/zero-slop?color=C15732"></a>
19
19
  <a href="https://www.npmjs.com/package/zero-slop"><img alt="npm downloads" src="https://img.shields.io/npm/dm/zero-slop?color=17634F"></a>
20
20
  <a href="https://github.com/manavmishra/ZeroSlop/stargazers"><img alt="GitHub stars" src="https://img.shields.io/github/stars/manavmishra/ZeroSlop?style=flat&color=C15732"></a>
@@ -120,7 +120,7 @@ We ran Zero Slop and three other open-source agent skills on our AI Slop test co
120
120
 
121
121
  ![Writing scores and local checks for a saved, same-model replay of the AI Slop test corpus; lower scores are better](assets/bench-search-rewrites.png)
122
122
 
123
- The [RAID+ audit](bench/raid-plus-corpus/README.md) asks a different question: how much default writing from different models is flagged as AI slop by Zero Slop? The test corpus contains 7,627 anonymous, user-generated transcripts:
123
+ Our [RAID+ audit](bench/raid-plus-corpus/README.md) scored 7,627 non-empty machine-generated abstracts from four models. The [INSCONE project](https://github.com/markstanl/INSCONE) extended RAID by generating new outputs from the original RAID prompts.
124
124
 
125
125
  | Model | Texts scored | Mean writing score ↓ | At or above 25 |
126
126
  |---|---:|---:|---:|
@@ -176,7 +176,7 @@ python3 scripts/slopscore.py --batch drafts/ --gate 25
176
176
  The CLI sends a file to the hosted editor without changing the file on disk:
177
177
 
178
178
  ```sh
179
- npx --yes zero-slop@2.12.29 deslop draft.md --genre professional
179
+ npx --yes zero-slop@2.12.31 deslop draft.md --genre professional
180
180
  ```
181
181
 
182
182
  Use `-` for stdin and `--json` for structured output. `--require-approved` prints the result but exits nonzero when review is needed. Requires Node.js 22+; offline `score` also needs Python 3. [CLI options and privacy](docs/cli.md).
package/SKILL.md CHANGED
@@ -2,7 +2,7 @@
2
2
  name: zero-slop
3
3
  license: MIT
4
4
  metadata:
5
- version: "2.12.29"
5
+ version: "2.12.31"
6
6
  author: manavmishra
7
7
  description: Edit drafts into natural prose, inspect AI-sounding patterns, or review how a specified audience might respond passage by passage. Zero Slop runs inside the user's existing AI assistant with local tools that protect source details. Use for humanizing or de-slopping writing, polishing outward-facing prose, social drafts, final editorial checks, or an explicit simulated reader review. Preserve facts, voice and format; reader simulations are hypotheses, not human feedback.
8
8
  ---
@@ -33,6 +33,22 @@ citations, and the ladder below orders the signals by measured strength.
33
33
 
34
34
  ## Hard rules (non-negotiable)
35
35
 
36
+ For an explicit character-cleanup request, use the bundled offline text-hygiene
37
+ helper at `scripts/text_hygiene.py` when available. It inspects formatting
38
+ characters and removes only a
39
+ small reviewed set of embedded artifacts in 16 common ASCII prose words.
40
+ Only sentence-shaped lines with conservative prose openers are eligible.
41
+ Recognized code, command syntax, URLs, formulas, quotations and multilingual
42
+ formatting are excluded; these guards are not a complete code parser. Preserve
43
+ uncertain text and use inspection when its purpose is unclear.
44
+ This is a separate local operation, not a replacement for contextual editing.
45
+ Its change record can reverse the exact cleanup. A scriptless assistant must
46
+ not claim byte-level cleanup it did not perform. Character cleanup does not
47
+ verify removal of statistical watermarks from ChatGPT, Claude, Gemini or any
48
+ other model. Preserve required disclosures and attribution.
49
+ In a host that routes local checks through the bundled MCP connector, use its
50
+ `clean_text` tool with the supplied text and optional `inspect: true` instead.
51
+
36
52
  1. **Fidelity.** Meaning, claims, and facts survive exactly. Never invent a
37
53
  number, name, anecdote, or experience — and experiential/interior claims
38
54
  count ("by test day it felt familiar", "I was terrified"): if the author
@@ -0,0 +1,6 @@
1
+ export type HygieneEdit = { start: number; end: number; original: string; replacement: ''; reason: string; outputStart: number; outputBinding: string };
2
+ export type HygieneResult = { text: string; edits: HygieneEdit[]; status: 'cleaned' | 'unchanged'; policyVersion: string; offsetUnit: 'utf16'; statisticalWatermark: { status: 'not_verified'; reason: string } };
3
+ export function cleanTextHygiene(text: string): HygieneResult;
4
+ export function inspectTextHygiene(text: string): Omit<HygieneResult, 'status'> & { status: 'inspected'; findings: Array<{ start: number; codePoint: string; disposition: 'removable' | 'preserved' }> };
5
+ export function reverseTextHygiene(text: string, edits: HygieneEdit[]): string;
6
+ export function createTextHygiene(): { cleanTextHygiene: typeof cleanTextHygiene; inspectTextHygiene: typeof inspectTextHygiene; reverseTextHygiene: typeof reverseTextHygiene };
@@ -0,0 +1,111 @@
1
+ /** Conservative copy-artifact cleanup. This never detects a provider watermark. */
2
+ export function createTextHygiene() {
3
+ const policyVersion = '2026-10-26.1';
4
+ const removable = /[\u200B\u2060\uFEFF]/g;
5
+ const words = new Set('a an and are clear draft hello in is of text the this to world writing'.split(' '));
6
+ const statisticalWatermark = { status: 'not_verified', reason: 'matching_detector_unavailable' };
7
+ function validate(text) {
8
+ if (typeof text !== 'string') throw new TypeError('Text must be a string.');
9
+ if (text.length > 200000) throw new RangeError('Text hygiene input exceeds 200000 UTF-16 units.');
10
+ }
11
+ // A change-detection fingerprint, not authentication or provenance evidence.
12
+ function binding(text) {
13
+ let first = 2166136261; let second = 2246822519;
14
+ for (let i = 0; i < text.length; i += 1) {
15
+ first = Math.imul(first ^ text.charCodeAt(i), 16777619) >>> 0;
16
+ second = Math.imul(second ^ text.charCodeAt(i), 3266489917) >>> 0;
17
+ }
18
+ return `${text.length}:${first}:${second}`;
19
+ }
20
+ function plannedEdits(text) {
21
+ if (/[<>]/.test(text)) return []; // Preserve unknown markup and multiline HTML as a whole.
22
+ const edits = []; let offset = 0; let fence = null; let frontmatter = false;
23
+ for (const line of text.split(/(?<=\n)/)) {
24
+ if (offset === 0 && /^\uFEFF?---\s*$/.test(line)) { frontmatter = true; offset += line.length; continue; }
25
+ if (frontmatter) { if (/^(?:---|\.\.\.)\s*$/.test(line)) frontmatter = false; offset += line.length; continue; }
26
+ const marker = line.match(/^\s*(`{3,}|~{3,})/);
27
+ if (marker) {
28
+ if (!fence) fence = { char: marker[1][0], length: marker[1].length };
29
+ else if (fence.char === marker[1][0] && marker[1].length >= fence.length && /^\s*(?:`+|~+)\s*$/.test(line)) fence = null;
30
+ offset += line.length; continue;
31
+ }
32
+ const plain = line.replace(removable, '');
33
+ // Sentence shape and a small set of prose openers keep unfenced commands
34
+ // and unknown fragments out of this deliberately conservative cleanup.
35
+ const protectedLine = fence || !/^\s*(?:the|this|hello|an?)\b/i.test(plain) || !/[.!?]\s*$/.test(plain)
36
+ || /[()[\]{};&]|(?:^|\s)--?[A-Za-z]/.test(plain)
37
+ || [...plain].some(char => (char.charCodeAt(0) < 32 && !'\t\n\r'.includes(char)) || char.charCodeAt(0) > 126)
38
+ || /^ {4}|^\t|[`"'<>$=\\:|]|https?:|www\.|@|\]\(|\b(?:identifier|variable|function|path|command|token|password|code|regex|formula|equation)\b/i.test(plain)
39
+ || /\b\w+[./_]\w+\b/.test(plain);
40
+ const ordinaryWords = plain.match(/[A-Za-z]+/g) || [];
41
+ if (!protectedLine && ordinaryWords.length >= 2) {
42
+ for (const word of line.matchAll(/[A-Za-z\u200B\u2060\uFEFF]+/g)) {
43
+ const cleaned = word[0].replace(removable, '');
44
+ if (!words.has(cleaned.toLowerCase())) continue;
45
+ for (let i = 1; i < word[0].length - 1; i += 1) {
46
+ const char = word[0][i];
47
+ if ('\u200B\u2060\uFEFF'.includes(char) && /[A-Za-z]/.test(word[0][i - 1]) && /[A-Za-z]/.test(word[0][i + 1])) {
48
+ const start = offset + word.index + i;
49
+ edits.push({ start, end: start + 1, original: char, replacement: '', reason: 'embedded_ascii_copy_artifact' });
50
+ }
51
+ }
52
+ }
53
+ }
54
+ offset += line.length;
55
+ }
56
+ return edits;
57
+ }
58
+ function cleanTextHygiene(text) {
59
+ validate(text);
60
+ const edits = plannedEdits(text);
61
+ const chunks = []; let cursor = 0;
62
+ for (const edit of edits) { chunks.push(text.slice(cursor, edit.start)); cursor = edit.end; }
63
+ chunks.push(text.slice(cursor));
64
+ const cleaned = chunks.join(''); const outputBinding = binding(cleaned);
65
+ for (let i = 0; i < edits.length; i += 1) {
66
+ edits[i].outputStart = edits[i].start - i;
67
+ edits[i].outputBinding = outputBinding;
68
+ }
69
+ return { text: cleaned, edits, status: edits.length ? 'cleaned' : 'unchanged', policyVersion,
70
+ offsetUnit: 'utf16', statisticalWatermark: { ...statisticalWatermark } };
71
+ }
72
+ function inspectTextHygiene(text) {
73
+ const result = cleanTextHygiene(text); const findings = [];
74
+ const actionable = new Set(result.edits.map(edit => edit.start));
75
+ for (let i = 0; i < text.length;) {
76
+ const cp = text.codePointAt(i);
77
+ if (cp === 0xAD || cp === 0xFEFF || (cp >= 0x200B && cp <= 0x200F)
78
+ || (cp >= 0x202A && cp <= 0x202E) || (cp >= 0x2060 && cp <= 0x2069)
79
+ || (cp >= 0xFE00 && cp <= 0xFE0F) || (cp >= 0xE0000 && cp <= 0xE01EF)) {
80
+ findings.push({ start: i, codePoint: `U+${cp.toString(16).toUpperCase().padStart(4, '0')}`,
81
+ disposition: actionable.has(i) ? 'removable' : 'preserved' });
82
+ }
83
+ i += cp > 0xFFFF ? 2 : 1;
84
+ }
85
+ return { ...result, text, findings, status: 'inspected' };
86
+ }
87
+ function reverseTextHygiene(text, edits) {
88
+ validate(text);
89
+ if (!Array.isArray(edits) || edits.length > 200000) throw new TypeError('Edits must be a bounded array.');
90
+ const chunks = []; let cursor = 0; let previous = -1; const outputBinding = binding(text);
91
+ for (let i = 0; i < edits.length; i += 1) {
92
+ const edit = edits[i];
93
+ if (!edit || typeof edit !== 'object' || !Number.isInteger(edit.start) || edit.start < 1 || edit.start > text.length + edits.length
94
+ || !Number.isInteger(edit.outputStart) || edit.outputStart < 0 || edit.outputStart > text.length
95
+ || edit.start <= previous || edit.end !== edit.start + 1
96
+ || !'\u200B\u2060\uFEFF'.includes(edit.original) || edit.original.length !== 1
97
+ || edit.replacement !== '' || edit.outputBinding !== outputBinding
98
+ || edit.outputStart !== edit.start - i) throw new Error('Cleanup plan does not match this text.');
99
+ previous = edit.start;
100
+ chunks.push(text.slice(cursor, edit.outputStart), edit.original); cursor = edit.outputStart;
101
+ }
102
+ chunks.push(text.slice(cursor));
103
+ return chunks.join('');
104
+ }
105
+ return Object.freeze({ cleanTextHygiene, inspectTextHygiene, reverseTextHygiene });
106
+ }
107
+
108
+ const helpers = createTextHygiene();
109
+ export const cleanTextHygiene = helpers.cleanTextHygiene;
110
+ export const inspectTextHygiene = helpers.inspectTextHygiene;
111
+ export const reverseTextHygiene = helpers.reverseTextHygiene;
package/bin/zero-slop.mjs CHANGED
@@ -12,6 +12,7 @@ import { homedir } from "node:os";
12
12
  import { dirname, join, parse, resolve } from "node:path";
13
13
  import { fileURLToPath } from "node:url";
14
14
  import { deslop, DeslopError, isApprovedResult, validateInput } from "./lib/deslop.mjs";
15
+ import { cleanTextHygiene, inspectTextHygiene } from "./lib/text-hygiene.mjs";
15
16
 
16
17
  const ROOT = resolve(dirname(fileURLToPath(import.meta.url)), "..");
17
18
  const PAYLOAD = ["SKILL.md", "references", "scripts", "data"];
@@ -37,6 +38,7 @@ function usage(v) {
37
38
  Usage
38
39
  npx zero-slop install [--harness <name>] [--dir <path>] [--force]
39
40
  npx zero-slop score <file|-> [-- <slopscore flags>]
41
+ npx zero-slop clean-text <file|-> [--inspect] [--json]
40
42
  npx zero-slop deslop <file|-> [--genre <name>] [--audience <reader>] [--json]
41
43
  [--require-approved] [--timeout <seconds>]
42
44
  npx zero-slop where
@@ -249,7 +251,7 @@ function parseDeslopArgs(argv) {
249
251
  json: Boolean(values["--json"]), requireApproved: Boolean(values["--require-approved"]), timeoutMs };
250
252
  }
251
253
 
252
- async function readDraft(file, signal) {
254
+ async function readDraft(file, signal, preserveBOM = false) {
253
255
  const maximum = 128 * 1024;
254
256
  let bytes;
255
257
  if (file === "-") {
@@ -288,7 +290,7 @@ async function readDraft(file, signal) {
288
290
  } finally { await handle?.close(); }
289
291
  }
290
292
  signal.throwIfAborted();
291
- try { return new TextDecoder("utf-8", { fatal: true }).decode(bytes); } catch {
293
+ try { return new TextDecoder("utf-8", { fatal: true, ignoreBOM: preserveBOM }).decode(bytes); } catch {
292
294
  throw new DeslopError("invalid_input", "The draft must be valid UTF-8; nothing was uploaded.");
293
295
  }
294
296
  }
@@ -348,6 +350,15 @@ async function runDeslop(argv, v) {
348
350
 
349
351
  async function main() {
350
352
  const argv = process.argv.slice(2);
353
+ if (argv[0] === "clean-text") {
354
+ const args = argv.slice(1);
355
+ const files = args.filter(arg => !["--inspect", "--json"].includes(arg));
356
+ if (files.length !== 1 || (files[0].startsWith("--"))) throw new Error("clean-text needs one file or -, plus optional --inspect and --json");
357
+ const text = await readDraft(files[0], new AbortController().signal, true);
358
+ const result = args.includes("--inspect") ? inspectTextHygiene(text) : cleanTextHygiene(text);
359
+ process.stdout.write(args.includes("--json") ? `${JSON.stringify(result)}\n` : result.text);
360
+ return 0;
361
+ }
351
362
  // Parse this command separately: no install flags or scorer passthrough can
352
363
  // accidentally become a remote-upload option.
353
364
  if (argv[0] === "deslop") {
package/docs/cli.md CHANGED
@@ -1,9 +1,25 @@
1
1
  # Zero Slop CLI
2
2
 
3
+ For an explicit local copy-artifact cleanup, use `npx zero-slop clean-text draft.md`.
4
+ Use `--inspect --json` to inspect without changing the returned text, or `--json`
5
+ with cleanup to receive reversible edits. The command never changes the source
6
+ file and accepts one regular UTF-8 file or stdin, up to 128 KiB.
7
+
8
+ This first policy removes only embedded U+200B, U+2060 and U+FEFF in a small
9
+ reviewed vocabulary of 16 common ASCII prose words. A line is eligible only
10
+ when it starts with The, This, Hello, A or An and ends with sentence punctuation.
11
+ Recognized code, command flags, operators, quotes, URLs, markup and non-ASCII
12
+ text are excluded. Other words and fragments stay unchanged. These guards are
13
+ conservative rules, not a complete parser for every programming language.
14
+ This deliberately limited cleanup does not verify statistical watermark removal.
15
+ The installed skill also includes the offline Python helper. Its offsets use
16
+ code points; the JavaScript result uses UTF-16 offsets. Change-detection bindings
17
+ are not authentication or provenance evidence.
18
+
3
19
  Use `deslop` for hosted editing with MCP parity. Use `score` for offline checks.
4
20
 
5
21
  ```sh
6
- npm install --global zero-slop@2.12.29
22
+ npm install --global zero-slop@2.12.31
7
23
  zero-slop deslop draft.md --genre professional
8
24
  zero-slop deslop - --genre email < draft.txt
9
25
  zero-slop deslop draft.md --json --require-approved
package/docs/rest-api.md CHANGED
@@ -22,9 +22,10 @@ curl --fail-with-body --max-time 75 --include https://mcp.zero-slop.ai/v1/deslop
22
22
 
23
23
  | Field | Accepted value |
24
24
  |---|---|
25
- | `text` | Required, nonempty string; up to 20,000 Unicode code points after trimming |
25
+ | `text` | Required, nonempty string; up to 20,000 Unicode code points. Editing trims the text first; `clean_text` preserves it and counts its full length. |
26
26
  | `genre` | `general` (default), `social`, `email`, `research`, or `professional` |
27
27
  | `audience` | Optional string, up to 200 Unicode code points after trimming |
28
+ | `operation` | Optional `"clean_text"` for conservative character cleanup without an editing-model call |
28
29
 
29
30
  Lengths count Unicode code points; a combined emoji or accented character can contain more than one.
30
31
  Send UTF-8 JSON in the body, never a URL query. The body limit is 128 KiB, including
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "zero-slop",
3
- "version": "2.12.29",
3
+ "version": "2.12.31",
4
4
  "description": "Find and fix canned AI writing while preserving the facts, voice, and format.",
5
5
  "mcpServers": {
6
6
  "zero-slop": {
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "zero-slop",
3
- "version": "2.12.29",
3
+ "version": "2.12.31",
4
4
  "description": "An editing skill, offline writing checks, and a CLI for the hosted Zero Slop MCP pipeline. Find stock AI phrasing, rewrite, and check source details.",
5
5
  "bin": {
6
6
  "zero-slop": "bin/zero-slop.mjs"
package/plugin.json CHANGED
@@ -1,7 +1,7 @@
1
1
  {
2
2
  "$schema": "https://agent-plugins.org/schemas/1.0.0/plugin.schema.json",
3
3
  "name": "zero-slop",
4
- "version": "2.12.29",
4
+ "version": "2.12.31",
5
5
  "description": "Find and fix canned AI writing while preserving the facts, voice, and format.",
6
6
  "author": {
7
7
  "name": "Manav Mishra",
@@ -0,0 +1,143 @@
1
+ #!/usr/bin/env python3
2
+ """Offline, conservative copy-artifact cleanup; not a watermark detector."""
3
+ import argparse
4
+ import hashlib
5
+ import json
6
+ import os
7
+ import re
8
+ import stat
9
+ import sys
10
+
11
+ POLICY_VERSION = "2026-10-26.1"
12
+ REMOVABLE = re.compile("[\u200b\u2060\ufeff]")
13
+ WORDS = frozenset("a an and are clear draft hello in is of text the this to world writing".split())
14
+ PROTECTED = re.compile(r'''^ {4}|^\t|[`"'<>$=\\:|]|https?:|www\.|@|\]\(|\b(?:identifier|variable|function|path|command|token|password|code|regex|formula|equation)\b|\b\w+[./_]\w+\b''', re.I)
15
+
16
+
17
+ def _validate(text):
18
+ if not isinstance(text, str):
19
+ raise TypeError("Text must be a string.")
20
+ if len(text.encode("utf-16-le", errors="surrogatepass")) // 2 > 200000:
21
+ raise ValueError("Text hygiene input exceeds 200000 UTF-16 units.")
22
+
23
+
24
+ def clean_text_hygiene(text):
25
+ _validate(text)
26
+ edits, offset, fence, frontmatter = [], 0, None, False
27
+ markup = "<" in text or ">" in text
28
+ parts = text.split("\n")
29
+ for line_index, part in enumerate(parts):
30
+ line = part + ("\n" if line_index < len(parts) - 1 else "")
31
+ if offset == 0 and re.fullmatch("\ufeff?---\\s*", line):
32
+ frontmatter = True
33
+ offset += len(line)
34
+ continue
35
+ if frontmatter:
36
+ if re.fullmatch(r"(?:---|\.\.\.)\s*", line):
37
+ frontmatter = False
38
+ offset += len(line)
39
+ continue
40
+ marker = re.match(r"\s*(`{3,}|~{3,})", line)
41
+ if marker:
42
+ char = marker[1][0]
43
+ if not fence:
44
+ fence = (char, len(marker[1]))
45
+ elif fence[0] == char and len(marker[1]) >= fence[1] and re.fullmatch(r"\s*(?:`+|~+)\s*", line):
46
+ fence = None
47
+ offset += len(line)
48
+ continue
49
+ plain = REMOVABLE.sub("", line)
50
+ # Preserve unfenced commands and fragments: only conservative prose
51
+ # openers with sentence-ending punctuation are eligible for cleanup.
52
+ protected = (markup or fence or not re.match(r"\s*(?:the|this|hello|an?)\b", plain, re.I)
53
+ or not re.search(r"[.!?]\s*$", plain)
54
+ or re.search(r"[()[\]{};&]|(?:^|\s)--?[A-Za-z]", plain)
55
+ or re.search(r"[^\x09\x0a\x0d\x20-\x7e]", plain) or PROTECTED.search(plain))
56
+ if not protected and len(re.findall(r"[A-Za-z]+", plain)) >= 2:
57
+ for match in re.finditer("[A-Za-z\u200b\u2060\ufeff]+", line):
58
+ word = match[0]
59
+ if REMOVABLE.sub("", word).lower() not in WORDS:
60
+ continue
61
+ for index in range(1, len(word) - 1):
62
+ if word[index] in "\u200b\u2060\ufeff" and re.fullmatch("[A-Za-z]", word[index - 1]) and re.fullmatch("[A-Za-z]", word[index + 1]):
63
+ start = offset + match.start() + index
64
+ edits.append({"start": start, "end": start + 1, "original": word[index], "replacement": "", "reason": "embedded_ascii_copy_artifact"})
65
+ offset += len(line)
66
+ chunks, cursor = [], 0
67
+ for edit in edits:
68
+ chunks.append(text[cursor:edit["start"]])
69
+ cursor = edit["end"]
70
+ chunks.append(text[cursor:])
71
+ cleaned = "".join(chunks)
72
+ digest = hashlib.sha256(cleaned.encode("utf-8", errors="surrogatepass")).hexdigest()
73
+ for index, edit in enumerate(edits):
74
+ edit.update(outputStart=edit["start"] - index, outputBinding=digest)
75
+ return {"text": cleaned, "edits": edits, "status": "cleaned" if edits else "unchanged", "policyVersion": POLICY_VERSION,
76
+ "offsetUnit": "codepoint", "statisticalWatermark": {"status": "not_verified", "reason": "matching_detector_unavailable"}}
77
+
78
+
79
+ def inspect_text_hygiene(text):
80
+ result = clean_text_hygiene(text)
81
+ actionable = {edit["start"] for edit in result["edits"]}
82
+ findings = []
83
+ for index, char in enumerate(text):
84
+ cp = ord(char)
85
+ if cp in (0xad, 0xfeff) or 0x200b <= cp <= 0x200f or 0x202a <= cp <= 0x202e or 0x2060 <= cp <= 0x2069 or 0xfe00 <= cp <= 0xfe0f or 0xe0000 <= cp <= 0xe01ef:
86
+ findings.append({"start": index, "codePoint": f"U+{cp:04X}", "disposition": "removable" if index in actionable else "preserved"})
87
+ return dict(result, text=text, findings=findings, status="inspected")
88
+
89
+
90
+ def reverse_text_hygiene(text, edits):
91
+ _validate(text)
92
+ if not isinstance(edits, list) or len(edits) > 200000:
93
+ raise ValueError("Edits must be a bounded array.")
94
+ digest = hashlib.sha256(text.encode("utf-8", errors="surrogatepass")).hexdigest()
95
+ chunks, cursor, previous = [], 0, -1
96
+ for index, edit in enumerate(edits):
97
+ if not isinstance(edit, dict):
98
+ raise ValueError("Cleanup plan does not match this text.")
99
+ start = edit.get("start")
100
+ output_start = edit.get("outputStart")
101
+ if (type(start) is not int or not 1 <= start <= len(text) + len(edits)
102
+ or type(output_start) is not int or not 0 <= output_start <= len(text)
103
+ or start <= previous or type(edit.get("end")) is not int
104
+ or edit["end"] != start + 1
105
+ or edit.get("original") not in ("\u200b", "\u2060", "\ufeff")
106
+ or edit.get("replacement") != "" or edit.get("outputBinding") != digest
107
+ or output_start != start - index):
108
+ raise ValueError("Cleanup plan does not match this text.")
109
+ previous = start
110
+ chunks.extend((text[cursor:output_start], edit["original"]))
111
+ cursor = output_start
112
+ chunks.append(text[cursor:])
113
+ return "".join(chunks)
114
+
115
+
116
+ def main():
117
+ parser = argparse.ArgumentParser(description=__doc__)
118
+ parser.add_argument("path", help="UTF-8 file or - for stdin")
119
+ parser.add_argument("--inspect", action="store_true")
120
+ parser.add_argument("--json", action="store_true")
121
+ args = parser.parse_args()
122
+ maximum = 128 * 1024
123
+ if args.path == "-":
124
+ payload = sys.stdin.buffer.read(maximum + 1)
125
+ else:
126
+ descriptor = os.open(args.path, os.O_RDONLY | getattr(os, "O_NONBLOCK", 0))
127
+ with os.fdopen(descriptor, "rb") as source:
128
+ if not stat.S_ISREG(os.fstat(source.fileno()).st_mode):
129
+ raise ValueError("Provide a regular UTF-8 file.")
130
+ payload = source.read(maximum + 1)
131
+ if len(payload) > maximum:
132
+ raise ValueError("Input exceeds 128 KiB.")
133
+ text = payload.decode("utf-8", errors="strict")
134
+ result = inspect_text_hygiene(text) if args.inspect else clean_text_hygiene(text)
135
+ print(json.dumps(result, ensure_ascii=True) if args.json else result["text"], end="\n" if args.json else "")
136
+
137
+
138
+ if __name__ == "__main__":
139
+ try:
140
+ main()
141
+ except (OSError, ValueError, TypeError):
142
+ print("Text cleanup failed: provide bounded, valid UTF-8 input.", file=sys.stderr)
143
+ raise SystemExit(2)