text-sanctifier 1.3.8 → 1.3.9

package/bin/text-sanctifier.js

@@ -378,18 +378,22 @@ function collectInputs(args, cwd, warn, aiConfig = false) {
 // --- Reading ---
 
 const utf8 = new TextDecoder('utf-8', { fatal: true, ignoreBOM: true });
+const utf8Lenient = new TextDecoder('utf-8', { ignoreBOM: true });
 
 /**
- * Decodes UTF-8, keeping a leading BOM. Returns null for binary or non-UTF-8 data.
+ * Decodes UTF-8, keeping a leading BOM. With `sniff`, data with a NUL in its first
+ * BINARY_SNIFF_BYTES is binary and not decoded. Bytes that are not UTF-8 become U+FFFD
+ * and set `lossy`, so one bad byte cannot hide the rest of a file from `check`.
  * @param {Uint8Array} bytes
- * @returns {{ text: string|null, reason?: string }}
+ * @param {boolean} sniff
+ * @returns {{ text: string|null, reason?: string, lossy?: boolean }}
  */
-function decode(bytes) {
-    if (bytes.subarray(0, BINARY_SNIFF_BYTES).includes(0)) return { text: null, reason: 'binary' };
+function decode(bytes, sniff) {
+    if (sniff && bytes.subarray(0, BINARY_SNIFF_BYTES).includes(0)) return { text: null, reason: 'binary' };
     try {
         return { text: utf8.decode(bytes) };
     } catch {
-        return { text: null, reason: 'not UTF-8' };
+        return { text: utf8Lenient.decode(bytes), lossy: true };
     }
 }
 
@@ -612,6 +616,8 @@ async function main(argv) {
 
     const cwd = process.cwd();
     let paths = opts.paths;
+    // --ai-config with no paths searches the current directory even when stdin is not a
+    // terminal, so a CI job with no stdin still scans the repository; `-` reads stdin.
     if (paths.length === 0 && opts.aiConfig) {
         paths = ['.'];
     } else if (paths.length === 0) {
@@ -627,6 +633,8 @@ async function main(argv) {
     let errors = pathErrors;
     /** @type {Array<{ path: string, reason: string }>} */
     const skipped = [];
+    /** @type {string[]} */
+    const notUtf8 = [];
 
     /**
      * Reads every input in order, calling `visit(label, text, input)` for each decodable one.
@@ -643,12 +651,24 @@ async function main(argv) {
                 errors++;
                 continue;
             }
-            const { text, reason } = decode(bytes);
+            // Files that look binary are skipped. Stdin and AI configuration files are text
+            // by intent, so a NUL in one is a finding, not a reason to skip it.
+            const { text, reason, lossy } = decode(bytes, input.path !== STDIN && !opts.aiConfig);
             if (text === null) {
                 skipped.push({ path: label, reason });
                 if (input.explicit) warn(`${label}: skipped (${reason})`);
                 continue;
             }
+            if (lossy) {
+                // fix never rewrites bytes it could not decode
+                if (opts.command === 'fix') {
+                    skipped.push({ path: label, reason: 'not UTF-8' });
+                    warn(`${label}: skipped (not UTF-8)`);
+                    continue;
+                }
+                notUtf8.push(label);
+                warn(`${label}: not valid UTF-8; checked with invalid bytes read as U+FFFD`);
+            }
             visit(label, text, input);
         }
     };
@@ -691,14 +711,17 @@ async function main(argv) {
 
         const kind = opts.aiConfig ? 'AI configuration file' : 'file';
         if (opts.format === 'json') {
-            const summary = { files, total, filesScanned: scanned, filesWithFindings: files.length, skipped };
+            const summary = { files, total, filesScanned: scanned, filesWithFindings: files.length, skipped, notUtf8 };
             if (opts.aiConfig) summary.hiddenRunCount = hiddenRunCount;
             out(JSON.stringify(summary, null, 2));
         } else if (opts.format === 'sarif') {
             out(JSON.stringify(sarifLog(files, { cwd, total, filesScanned: scanned, skipped, succeeded: errors === 0 }), null, 2));
         } else {
             for (const line of lines) out(line);
-            if (opts.aiConfig && scanned === 0 && skipped.length === 0) out('No AI configuration files found.');
+            if (opts.aiConfig && scanned === 0 && skipped.length === 0) {
+                out('No AI configuration files found.');
+                if (opts.paths.length === 0 && !process.stdin.isTTY) out('To check piped text, pass "-": text-sanctifier check --ai-config -');
+            }
             else if (total === 0) out(`No ${opts.aiConfig ? 'hidden text, ' : ''}invisible or control characters in ${plural(scanned, kind)}.`);
             else out(`${plural(total, 'finding')}${opts.aiConfig ? ` (${plural(hiddenRunCount, 'hidden run')})` : ''} in ${files.length} of ${plural(scanned, kind)}.`);
         }

package/dist/text-sanctifier.min.js

@@ -3,4 +3,4 @@ var y=/[\u0000-\u0008\u000B\u000C\u000E-\u001F\u007F-\u009F\u061C\u200E\u200F\u2
 `){return e.replace(Q,r)}var re=/[ \t]*(\n+)[ \t]*/g;function ne(e){return e.replace(re,"$1")}var te=/\n{2,}/g,ie=/\n{3,}/g;function oe(e,r){return r?e.replace(ie,`
 
 `):e.replace(te,`
-`)}var ue=/ {2,}/g;function se(e){return e.replace(ue," ")}function le(e){return e.replace(y,"")}function ae(e,r={}){if(typeof e!="string")throw new TypeError("revealInvisible expects a string input.");let n=r&&typeof r.format=="function"?r.format:i=>"<U+"+i.toString(16).toUpperCase().padStart(4,"0")+">",u="",o=0;for(let i of C(e,{maxFindings:1/0}).findings)u+=e.slice(o,i.offset)+n(i.codePoint,i.category),o=i.offset+(i.codePoint>65535?2:1);return u+e.slice(o)}var fe="1.3.8";export{fe as VERSION,b as getNewlineStats,K as getRecommendedSanctifierOptions,j as inspectText,C as inspectTextDetailed,ae as revealInvisible,T as sanctifyKeyboardOnly,N as sanctifyKeyboardOnlyEmoji,w as sanctifyLoose,L as sanctifyPasted,R as sanctifyStrict,W as summonSanctifier};
+`)}var ue=/ {2,}/g;function se(e){return e.replace(ue," ")}function le(e){return e.replace(y,"")}function ae(e,r={}){if(typeof e!="string")throw new TypeError("revealInvisible expects a string input.");let n=r&&typeof r.format=="function"?r.format:i=>"<U+"+i.toString(16).toUpperCase().padStart(4,"0")+">",u="",o=0;for(let i of C(e,{maxFindings:1/0}).findings)u+=e.slice(o,i.offset)+n(i.codePoint,i.category),o=i.offset+(i.codePoint>65535?2:1);return u+e.slice(o)}var fe="1.3.9";export{fe as VERSION,b as getNewlineStats,K as getRecommendedSanctifierOptions,j as inspectText,C as inspectTextDetailed,ae as revealInvisible,T as sanctifyKeyboardOnly,N as sanctifyKeyboardOnlyEmoji,w as sanctifyLoose,L as sanctifyPasted,R as sanctifyStrict,W as summonSanctifier};

package/dist/types/hiddenText.d.ts

@@ -13,7 +13,10 @@ export function decodeTagText(text: string): string;
  * legitimate and listed under `benign` instead of `runs`:
  *  - the subdivision flags of England, Scotland and Wales: U+1F3F4 followed by the
  *    tags for "gbeng", "gbsct" or "gbwls" and U+E007F
- *  - a single U+FE0E or U+FE0F (text or emoji presentation) after another character
+ *  - a single U+FE0E or U+FE0F (text or emoji presentation) after an
+ *    Extended_Pictographic character, or between #, * or a digit and U+20E3 (a keycap),
+ *    the same rule as text-sanctifier/context. After a letter it is a run: one selector
+ *    per letter can carry a bit each.
  *
  * `line` and `column` are counted as in inspectTextDetailed. Every run and benign
  * span is counted; only the first `maxRuns` of each are listed. With the caps set,

package/dist/types/sanctifyText.d.ts

@@ -52,7 +52,7 @@ export function sanctifyPasted(text: string): string;
  * Brutal text normalizer and invisible trash scrubber,
  * configurable to kill whatever ghosts you want dead.
  *
- * ⚠️ Note: This is plain-text normalization/filtering — not an HTML/XSS sanitizer.
+ * Note: This is plain-text normalization/filtering — not an HTML/XSS sanitizer.
  *
  * @param {string} text
  * @param {boolean} [purgeInvisibleChars=false] - Remove ZWSP, NBSP, bidi, tags, variation selectors, etc.

package/dist/types/version.generated.d.ts

@@ -1,2 +1,2 @@
 /** The installed version of text-sanctifier, as in its package.json. */
-export const VERSION: "1.3.8";
+export const VERSION: "1.3.9";

package/package.json

@@ -1,83 +1,89 @@
-{
-  "name": "text-sanctifier",
-  "version": "1.3.8",
-  "type": "module",
-  "description": "A brutal text normalizer and invisible trash scrubber for modern web projects.",
-  "engines": {
-    "node": ">=18"
-  },
-  "main": "./src/index.js",
-  "module": "./src/index.js",
-  "browser": "./dist/text-sanctifier.min.js",
-  "bin": {
-    "text-sanctifier": "bin/text-sanctifier.js"
-  },
-  "files": [
-    "src",
-    "bin",
-    "dist/text-sanctifier.min.js",
-    "dist/types",
-    "LICENSE",
-    "README.md",
-    "ADDITIONAL_TERMS.md",
-    "robots.txt"
-  ],
-  "exports": {
-    ".": {
-      "types": "./dist/types/index.d.ts",
-      "default": "./src/index.js"
-    },
-    "./hidden": {
-      "types": "./dist/types/hiddenText.d.ts",
-      "default": "./src/hiddenText.js"
-    },
-    "./context": {
-      "types": "./dist/types/context.d.ts",
-      "default": "./src/context.js"
-    },
-    "./browser": {
-      "types": "./dist/types/index.d.ts",
-      "default": "./dist/text-sanctifier.min.js"
-    }
-  },
-  "types": "./dist/types/index.d.ts",
-  "sideEffects": false,
-  "scripts": {
-    "gen": "node scripts/gen-categories.js && node scripts/gen-version.js && node scripts/gen-virama.js",
-    "gen:check": "node scripts/gen-categories.js --check && node scripts/gen-version.js --check && node scripts/gen-virama.js --check",
-    "build": "node scripts/build.js",
-    "build:types": "tsc -p tsconfig.types.json",
-    "test:types": "tsc -p tests/types/tsconfig.json",
-    "test": "node tests/sanctifyText.test.js && node tests/inspectTextDetailed.test.js && node --expose-gc tests/hiddenText.test.js && node --expose-gc tests/context.test.js && node tests/cli.test.js",
-    "test-min": "node tests/sanctifyText.test.min.js",
-    "ci": "npm run gen:check && npm run test && npm run build && npm run size && npm run build:types && npm run test:types && npm run test-min",
-    "size": "node scripts/check-size.js"
-  },
-  "keywords": [
-    "text",
-    "sanitize",
-    "normalize",
-    "invisible",
-    "scrubber",
-    "unicode",
-    "clean",
-    "sanctify"
-  ],
-  "author": "WATT3D <WATT3D@protonmail.com>",
-  "license": "SEE LICENSE IN LICENSE",
-  "repository": {
-    "type": "git",
-    "url": "git+https://github.com/iWhatty/text-sanctifier.git"
-  },
-  "bugs": {
-    "url": "https://github.com/iWhatty/text-sanctifier/issues"
-  },
-  "homepage": "https://github.com/iWhatty/text-sanctifier#readme",
-  "directories": {
-    "test": "tests"
-  },
-  "devDependencies": {
-    "esbuild": "^0.25.3",
-    "typescript": "^5.9.3"
-  }
-}
+{
+  "name": "text-sanctifier",
+  "version": "1.3.9",
+  "type": "module",
+  "description": "A brutal text normalizer and invisible trash scrubber for modern web projects.",
+  "engines": {
+    "node": ">=18"
+  },
+  "main": "./src/index.js",
+  "module": "./src/index.js",
+  "browser": "./dist/text-sanctifier.min.js",
+  "bin": {
+    "text-sanctifier": "bin/text-sanctifier.js"
+  },
+  "files": [
+    "src",
+    "bin",
+    "dist/text-sanctifier.min.js",
+    "dist/types",
+    "LICENSE",
+    "README.md",
+    "ADDITIONAL_TERMS.md",
+    "robots.txt"
+  ],
+  "exports": {
+    ".": {
+      "types": "./dist/types/index.d.ts",
+      "default": "./src/index.js"
+    },
+    "./hidden": {
+      "types": "./dist/types/hiddenText.d.ts",
+      "default": "./src/hiddenText.js"
+    },
+    "./context": {
+      "types": "./dist/types/context.d.ts",
+      "default": "./src/context.js"
+    },
+    "./browser": {
+      "types": "./dist/types/index.d.ts",
+      "default": "./dist/text-sanctifier.min.js"
+    }
+  },
+  "types": "./dist/types/index.d.ts",
+  "sideEffects": false,
+  "scripts": {
+    "gen": "node scripts/gen-categories.js && node scripts/gen-version.js && node scripts/gen-virama.js && node scripts/gen-docs.js",
+    "gen:check": "node scripts/gen-categories.js --check && node scripts/gen-version.js --check && node scripts/gen-virama.js --check && node scripts/gen-docs.js --check",
+    "build": "node scripts/build.js",
+    "build:types": "tsc -p tsconfig.types.json",
+    "test:types": "tsc -p tests/types/tsconfig.json",
+    "test": "node tests/sanctifyText.test.js && node tests/inspectTextDetailed.test.js && node --expose-gc tests/hiddenText.test.js && node --expose-gc tests/context.test.js && node tests/cli.test.js && node tests/release.test.js",
+    "test-min": "node tests/sanctifyText.test.min.js",
+    "check:self": "node bin/text-sanctifier.js check .",
+    "check:names": "node scripts/check-names.js",
+    "ci": "npm run gen:check && npm run check:names && npm run check:self && npm run test && npm run build && npm run size && npm run build:types && npm run test:types && npm run test-min",
+    "size": "node scripts/check-size.js",
+    "bench": "node --expose-gc scripts/bench.js",
+    "release:prep": "node scripts/release.js prep",
+    "release:verify": "node scripts/release.js verify",
+    "hooks:install": "git config core.hooksPath scripts/hooks"
+  },
+  "keywords": [
+    "text",
+    "sanitize",
+    "normalize",
+    "invisible",
+    "scrubber",
+    "unicode",
+    "clean",
+    "sanctify"
+  ],
+  "author": "WATT3D <WATT3D@protonmail.com>",
+  "license": "SEE LICENSE IN LICENSE",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/iWhatty/text-sanctifier.git"
+  },
+  "bugs": {
+    "url": "https://github.com/iWhatty/text-sanctifier/issues"
+  },
+  "homepage": "https://github.com/iWhatty/text-sanctifier#readme",
+  "directories": {
+    "test": "tests"
+  },
+  "devDependencies": {
+    "esbuild": "^0.25.3",
+    "typescript": "^5.9.3"
+  }
+}

package/robots.txt

@@ -1,79 +1,79 @@
-# WATT3D — AI scraping policy
-#
-# This file enumerates known AI / LLM training and inference crawlers
-# that the WATT3D AI Training Rider does not consent to. See
-# ./ADDITIONAL_TERMS.md for the legal scope and definitions.
-#
-# Honoring robots.txt is voluntary. The rider applies whether or not
-# a bot respects this file. The enumeration is non-exhaustive — bots
-# not listed here are still bound by the rider if they perform
-# Training Use as defined in ADDITIONAL_TERMS.md §1.
-
-User-agent: GPTBot
-Disallow: /
-
-User-agent: ChatGPT-User
-Disallow: /
-
-User-agent: OAI-SearchBot
-Disallow: /
-
-User-agent: anthropic-ai
-Disallow: /
-
-User-agent: ClaudeBot
-Disallow: /
-
-User-agent: Claude-Web
-Disallow: /
-
-User-agent: Claude-SearchBot
-Disallow: /
-
-User-agent: Google-Extended
-Disallow: /
-
-User-agent: Applebot-Extended
-Disallow: /
-
-User-agent: Meta-ExternalAgent
-Disallow: /
-
-User-agent: Meta-ExternalFetcher
-Disallow: /
-
-User-agent: FacebookBot
-Disallow: /
-
-User-agent: CCBot
-Disallow: /
-
-User-agent: PerplexityBot
-Disallow: /
-
-User-agent: Perplexity-User
-Disallow: /
-
-User-agent: cohere-ai
-Disallow: /
-
-User-agent: Bytespider
-Disallow: /
-
-User-agent: Amazonbot
-Disallow: /
-
-User-agent: Omgilibot
-Disallow: /
-
-User-agent: AI2Bot
-Disallow: /
-
-User-agent: Diffbot
-Disallow: /
-
-# Everyone else: full access. Search engines and human-driven crawlers
-# remain welcome.
-
-User-agent: *
-Allow: /
+# WATT3D — AI scraping policy
+#
+# This file enumerates known AI / LLM training and inference crawlers
+# that the WATT3D AI Training Rider does not consent to. See
+# ./ADDITIONAL_TERMS.md for the legal scope and definitions.
+#
+# Honoring robots.txt is voluntary. The rider applies whether or not
+# a bot respects this file. The enumeration is non-exhaustive — bots
+# not listed here are still bound by the rider if they perform
+# Training Use as defined in ADDITIONAL_TERMS.md §1.
+
+User-agent: GPTBot
+Disallow: /
+
+User-agent: ChatGPT-User
+Disallow: /
+
+User-agent: OAI-SearchBot
+Disallow: /
+
+User-agent: anthropic-ai
+Disallow: /
+
+User-agent: ClaudeBot
+Disallow: /
+
+User-agent: Claude-Web
+Disallow: /
+
+User-agent: Claude-SearchBot
+Disallow: /
+
+User-agent: Google-Extended
+Disallow: /
+
+User-agent: Applebot-Extended
+Disallow: /
+
+User-agent: Meta-ExternalAgent
+Disallow: /
+
+User-agent: Meta-ExternalFetcher
+Disallow: /
+
+User-agent: FacebookBot
+Disallow: /
+
+User-agent: CCBot
+Disallow: /
+
+User-agent: PerplexityBot
+Disallow: /
+
+User-agent: Perplexity-User
+Disallow: /
+
+User-agent: cohere-ai
+Disallow: /
+
+User-agent: Bytespider
+Disallow: /
+
+User-agent: Amazonbot
+Disallow: /
+
+User-agent: Omgilibot
+Disallow: /
+
+User-agent: AI2Bot
+Disallow: /
+
+User-agent: Diffbot
+Disallow: /
+
+# Everyone else: full access. Search engines and human-driven crawlers
+# remain welcome.
+
+User-agent: *
+Allow: /