slopfix 0.0.0-stage → 0.1.1

package/package.json

@@ -1,6 +1,58 @@
 {
   "name": "slopfix",
-  "version": "0.0.0-stage",
-  "stub": true,
-  "description": "Temporary package placeholder for staged publishing"
-}
+  "version": "0.1.1",
+  "description": "Find and fix AI writing tells (em dashes, \"it's not X, it's Y\", delve, chatbot leftovers) in English and French. Zero dependencies. CLI, library, GitHub Action.",
+  "type": "module",
+  "main": "./src/index.js",
+  "types": "./index.d.ts",
+  "exports": {
+    ".": {
+      "types": "./index.d.ts",
+      "default": "./src/index.js"
+    },
+    "./package.json": "./package.json"
+  },
+  "bin": {
+    "slopfix": "bin/slopfix.js"
+  },
+  "files": [
+    "bin",
+    "src",
+    "index.d.ts"
+  ],
+  "scripts": {
+    "test": "node --test",
+    "lint:docs": "node bin/slopfix.js README.md README.fr.md CONTRIBUTING.md CHANGELOG.md skill/SKILL.md",
+    "demo": "node examples/make-examples.js && node bin/slopfix.js examples"
+  },
+  "keywords": [
+    "ai",
+    "llm",
+    "chatgpt",
+    "claude",
+    "slop",
+    "ai-slop",
+    "ai-writing",
+    "humanizer",
+    "linter",
+    "prose",
+    "writing",
+    "markdown",
+    "em-dash",
+    "cli",
+    "github-action"
+  ],
+  "author": "Zwarmex",
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/Zwarmex/slopfix.git"
+  },
+  "homepage": "https://zwarmex.github.io/slopfix/",
+  "bugs": {
+    "url": "https://github.com/Zwarmex/slopfix/issues"
+  },
+  "engines": {
+    "node": ">=18"
+  }
+}

package/src/format.js

@@ -0,0 +1,110 @@
+// Output formats for the CLI: pretty (humans), compact (grep), json (tools), github (Actions annotations).
+
+const useColor = !process.env.NO_COLOR && (process.env.FORCE_COLOR || process.stdout.isTTY);
+const paint = (code) => (s) => (useColor ? `\x1b[${code}m${s}\x1b[0m` : String(s));
+export const c = {
+  red: paint(31),
+  yellow: paint(33),
+  blue: paint(34),
+  green: paint(32),
+  dim: paint(2),
+  bold: paint(1),
+  underline: paint(4),
+};
+
+const SEVERITY_COLOR = { error: c.red, warning: c.yellow, info: c.blue };
+
+function scoreColor(score) {
+  if (score < 10) return c.green;
+  if (score < 30) return c.yellow;
+  return c.red;
+}
+
+function truncate(s, n) {
+  const flat = s.replace(/\s+/g, ' ');
+  return flat.length > n ? flat.slice(0, n - 1) + '…' : flat;
+}
+
+export function pretty(results, { quiet = false } = {}) {
+  const out = [];
+  const totals = { error: 0, warning: 0, info: 0, fixable: 0 };
+
+  for (const r of results) {
+    const issues = r.issues.filter((i) => !quiet || i.severity !== 'info');
+    for (const i of r.issues) {
+      totals[i.severity]++;
+      if (i.fix) totals.fixable++;
+    }
+    const header = `${c.underline(r.file)}  ${c.dim(`${r.language} · ${r.words} words ·`)} ${scoreColor(r.score)(`slop score ${r.score}/100 (${r.label})`)}`;
+    if (issues.length === 0) {
+      out.push(header, '');
+      continue;
+    }
+    out.push(header);
+    const posWidth = Math.max(...issues.map((i) => `${i.line}:${i.column}`.length));
+    for (const i of issues) {
+      const pos = `${i.line}:${i.column}`.padEnd(posWidth);
+      const sev = SEVERITY_COLOR[i.severity](i.severity.padEnd(7));
+      const fixNote = i.fix ? c.green(` → ${i.fix.label || `"${i.fix.text.trim() || i.fix.text}"`}`) : '';
+      out.push(`  ${c.dim(pos)}  ${sev}  ${c.bold(truncate(i.quote, 50))}${fixNote}  ${c.dim(i.message)}  ${c.dim(i.ruleId)}`);
+    }
+    out.push('');
+  }
+
+  const problems = totals.error + totals.warning + totals.info;
+  if (problems === 0) {
+    out.push(c.green('✔ No AI writing tells found.'));
+  } else {
+    const parts = [`${totals.error} errors`, `${totals.warning} warnings`, `${totals.info} info`];
+    let line = `${totals.error ? c.red('✖') : c.yellow('!')} ${problems} tells (${parts.join(', ')})`;
+    if (totals.fixable) line += c.dim(` · ${totals.fixable} fixable with --fix`);
+    out.push(line);
+  }
+  return out.join('\n');
+}
+
+export function compact(results) {
+  const out = [];
+  for (const r of results) {
+    for (const i of r.issues) out.push(`${r.file}:${i.line}:${i.column}: ${i.severity} ${i.message} "${truncate(i.quote, 60)}" [${i.ruleId}]`);
+  }
+  return out.join('\n');
+}
+
+export function json(results) {
+  return JSON.stringify(
+    results.map((r) => ({ ...r, issues: r.issues.map(({ index, length, ...rest }) => ({ ...rest, offset: index, length })) })),
+    null,
+    2,
+  );
+}
+
+function escapeData(s) {
+  return String(s).replace(/%/g, '%25').replace(/\r/g, '%0D').replace(/\n/g, '%0A');
+}
+
+function escapeProperty(s) {
+  return escapeData(s).replace(/:/g, '%3A').replace(/,/g, '%2C');
+}
+
+const GITHUB_LEVEL = { error: 'error', warning: 'warning', info: 'notice' };
+
+export function github(results) {
+  const out = [];
+  for (const r of results) {
+    for (const i of r.issues) {
+      const props = [
+        `file=${escapeProperty(r.file)}`,
+        `line=${i.line}`,
+        `col=${i.column}`,
+        `endLine=${i.endLine}`,
+        `endColumn=${i.endColumn}`,
+        `title=${escapeProperty(`slopfix (${i.ruleId})`)}`,
+      ].join(',');
+      out.push(`::${GITHUB_LEVEL[i.severity]} ${props}::${escapeData(`"${truncate(i.quote, 60)}" ${i.message}`)}`);
+    }
+  }
+  return out.join('\n');
+}
+
+export const formatters = { pretty, compact, json, github };

package/src/index.js

@@ -0,0 +1,182 @@
+import common from './rules/common.js';
+import en from './rules/en.js';
+import fr from './rules/fr.js';
+import { phraseRule } from './rules/helpers.js';
+import { mask } from './mask.js';
+
+export { mask } from './mask.js';
+
+export const rules = [...common, ...en, ...fr];
+export const languages = ['en', 'fr'];
+export const weights = { error: 5, warning: 2, info: 1 };
+
+const SEVERITIES = new Set(['error', 'warning', 'info']);
+
+const STOPWORDS = {
+  en: ['the', 'and', 'is', 'of', 'to', 'in', 'that', 'it', 'for', 'with', 'you', 'are', 'this', 'not', 'be', 'on', 'your', 'can', 'what', 'how'],
+  fr: ['le', 'la', 'les', 'des', 'est', 'et', 'une', 'un', 'du', 'que', 'pour', 'dans', 'pas', 'vous', 'nous', 'avec', 'sur', 'qui', 'au', 'ce'],
+};
+
+export const defaults = {
+  lang: 'auto',
+  rules: {},
+  ban: [],
+  dashReplacement: ', ',
+  markdown: true,
+};
+
+/** Guesses the language of a text from its most common short words. Returns a language code. */
+export function detectLanguage(text) {
+  const counts = Object.fromEntries(Object.keys(STOPWORDS).map((l) => [l, 0]));
+  const sets = Object.fromEntries(Object.entries(STOPWORDS).map(([l, w]) => [l, new Set(w)]));
+  for (const word of text.toLowerCase().match(/\p{L}+/gu) || []) {
+    for (const l in sets) if (sets[l].has(word)) counts[l]++;
+  }
+  return Object.entries(counts).sort((a, b) => b[1] - a[1])[0][0];
+}
+
+function normalizeOptions(options = {}) {
+  return { ...defaults, ...options, rules: { ...defaults.rules, ...(options.rules || {}) } };
+}
+
+function banRule(ban) {
+  if (!ban || ban.length === 0) return [];
+  const phrases = ban.map((b) => (typeof b === 'string' ? b : b.replacement !== undefined ? [b.phrase, b.replacement] : b.phrase));
+  return [
+    phraseRule({
+      id: 'custom/ban',
+      lang: '*',
+      category: 'custom',
+      severity: 'warning',
+      message: 'Banned by your slopfix config.',
+      phrases,
+    }),
+  ];
+}
+
+function activeRules(options, language) {
+  const out = [];
+  for (const rule of [...rules, ...banRule(options.ban)]) {
+    if (rule.lang !== '*' && options.lang !== 'all' && rule.lang !== language) continue;
+    const setting = options.rules[rule.id];
+    if (setting === 'off' || setting === false) continue;
+    out.push(SEVERITIES.has(setting) ? { ...rule, severity: setting } : rule);
+  }
+  return out;
+}
+
+function lineIndex(text) {
+  const starts = [0];
+  for (let i = 0; i < text.length; i++) if (text[i] === '\n') starts.push(i + 1);
+  return (offset) => {
+    let lo = 0;
+    let hi = starts.length - 1;
+    while (lo < hi) {
+      const mid = (lo + hi + 1) >> 1;
+      if (starts[mid] <= offset) lo = mid;
+      else hi = mid - 1;
+    }
+    return { line: lo + 1, column: offset - starts[lo] + 1 };
+  };
+}
+
+/** Converts weighted hits into a 0 to 100 score, where 0 is clean and 100 is pure slop. */
+export function computeScore(issues, words) {
+  const weighted = issues.reduce((sum, i) => sum + weights[i.severity], 0);
+  const per100 = (weighted / Math.max(words, 50)) * 100;
+  // Saturating curve: one em dash per 100 words is about 12, a typical chatbot answer lands between 30 and 60.
+  return Math.round(100 * (1 - Math.exp(-per100 / 15)));
+}
+
+export function scoreLabel(score) {
+  if (score < 10) return 'clean';
+  if (score < 30) return 'a few tells';
+  if (score < 60) return 'sloppy';
+  return 'pure slop';
+}
+
+/**
+ * Finds AI writing tells in a text.
+ * @returns {{ language: string, words: number, score: number, label: string, issues: object[] }}
+ */
+export function lint(text, options) {
+  const opts = normalizeOptions(options);
+  const masked = mask(text, { markdown: opts.markdown });
+  const language = opts.lang === 'auto' || opts.lang === 'all' ? detectLanguage(masked) : opts.lang;
+  const ctx = { options: opts, language };
+  const position = lineIndex(text);
+  const issues = [];
+
+  for (const rule of activeRules(opts, language)) {
+    for (const hit of rule.find(masked, ctx)) {
+      const start = position(hit.index);
+      const end = position(hit.index + hit.length);
+      issues.push({
+        ruleId: rule.id,
+        category: rule.category,
+        severity: rule.severity,
+        message: hit.message || rule.message,
+        quote: hit.quote ?? text.slice(hit.index, hit.index + hit.length),
+        index: hit.index,
+        length: hit.length,
+        line: start.line,
+        column: start.column,
+        endLine: end.line,
+        endColumn: end.column,
+        fix:
+          hit.fix === undefined || hit.fix === null
+            ? undefined
+            : { range: [hit.index, hit.index + (hit.fixLength ?? hit.length)], text: hit.fix, label: hit.fixLabel ?? (hit.fix === '' ? '(remove)' : undefined) },
+      });
+    }
+  }
+
+  // Longest span first at each position, then drop tells that sit entirely inside another one
+  // ("crucial" inside "plays a crucial role" is one tell, not two).
+  issues.sort((a, b) => a.index - b.index || b.length - a.length || a.ruleId.localeCompare(b.ruleId));
+  const kept = [];
+  for (const issue of issues) {
+    const outer = kept.find((k) => issue.index >= k.index && issue.index + issue.length <= k.index + k.length);
+    if (!outer) kept.push(issue);
+  }
+  issues.length = 0;
+  issues.push(...kept);
+  const words = (masked.match(/\p{L}+/gu) || []).length;
+  const score = computeScore(issues, words);
+  return { language, words, score, label: scoreLabel(score), issues };
+}
+
+function applyFixes(text, issues) {
+  const fixes = issues
+    .filter((i) => i.fix)
+    .map((i) => i.fix)
+    .sort((a, b) => a.range[0] - b.range[0]);
+  const kept = [];
+  let lastEnd = -1;
+  for (const f of fixes) {
+    if (f.range[0] < lastEnd) continue;
+    kept.push(f);
+    lastEnd = f.range[1];
+  }
+  let out = text;
+  for (const f of kept.reverse()) out = out.slice(0, f.range[0]) + f.text + out.slice(f.range[1]);
+  return { output: out, applied: kept.length };
+}
+
+/**
+ * Applies every safe autofix. Runs a few passes because one fix can reveal another.
+ * @returns {{ output: string, fixed: number, result: ReturnType<typeof lint> }}
+ */
+export function fix(text, options) {
+  let output = text;
+  let fixed = 0;
+  for (let pass = 0; pass < 5; pass++) {
+    const { output: next, applied } = applyFixes(output, lint(output, options).issues);
+    if (applied === 0) break;
+    output = next;
+    fixed += applied;
+  }
+  return { output, fixed, result: lint(output, options) };
+}
+
+export default { lint, fix, rules, detectLanguage, computeScore, scoreLabel, mask, languages, defaults };

package/src/mask.js

@@ -0,0 +1,65 @@
+// Hides the parts of a document that are not prose (code, URLs, HTML, front matter, disabled regions)
+// by replacing them with spaces. Offsets stay identical, so issues map back to the original text.
+
+const DISABLE = /<!--\s*slopfix-disable\s*-->/g;
+const ENABLE = /<!--\s*slopfix-enable\s*-->/g;
+const DISABLE_NEXT_LINE = /<!--\s*slopfix-disable-next-line\s*-->/g;
+
+function blank(chars, start, end) {
+  for (let i = start; i < end && i < chars.length; i++) {
+    if (chars[i] !== '\n') chars[i] = ' ';
+  }
+}
+
+function blankAll(chars, text, re, group = 0) {
+  for (const m of text.matchAll(re)) {
+    const start = group ? m.index + m[0].indexOf(m[group]) : m.index;
+    const len = group ? m[group].length : m[0].length;
+    blank(chars, start, start + len);
+  }
+}
+
+export function disabledRanges(text) {
+  const ranges = [];
+  for (const m of text.matchAll(DISABLE)) {
+    ENABLE.lastIndex = m.index;
+    const end = ENABLE.exec(text);
+    ranges.push([m.index, end ? end.index + end[0].length : text.length]);
+  }
+  for (const m of text.matchAll(DISABLE_NEXT_LINE)) {
+    const lineEnd = text.indexOf('\n', m.index);
+    if (lineEnd === -1) continue;
+    const nextEnd = text.indexOf('\n', lineEnd + 1);
+    ranges.push([m.index, nextEnd === -1 ? text.length : nextEnd]);
+  }
+  return ranges;
+}
+
+export function mask(text, { markdown = true } = {}) {
+  const chars = text.split('');
+
+  for (const [start, end] of disabledRanges(text)) blank(chars, start, end);
+
+  if (markdown) {
+    // Front matter
+    const fm = /^---\r?\n[\s\S]*?\r?\n---\r?\n/.exec(text);
+    if (fm) blank(chars, 0, fm[0].length);
+    // Fenced code blocks (``` or ~~~), unterminated fences run to the end
+    blankAll(chars, text, /^([ \t]*)(`{3,}|~{3,})[^\n]*\n[\s\S]*?(?:^\1?\2[ \t]*$|(?![\s\S]))/gm);
+    // Inline code, never across a blank line
+    blankAll(chars, text, /(`+)(?:(?!\1)(?!\n[ \t]*\n)[\s\S])+?\1/g);
+    // HTML comments and tags
+    blankAll(chars, text, /<!--[\s\S]*?-->/g);
+    blankAll(chars, text, /<\/?[a-zA-Z][^>\n]*>/g);
+    // Link and image destinations: keep the label, hide the URL
+    blankAll(chars, text, /\]\(([^)\s]+(?:\s+"[^"]*")?)\)/g, 1);
+    // Reference-style link definitions
+    blankAll(chars, text, /^[ \t]*\[[^\]]+\]:[ \t]*\S+.*$/gm);
+  }
+
+  // Bare URLs and emails in any format
+  blankAll(chars, text, /\bhttps?:\/\/[^\s<>()]+/g);
+  blankAll(chars, text, /\b[\w.+-]+@[\w-]+\.[\w.-]+\b/g);
+
+  return chars.join('');
+}

package/src/rules/common.js

@@ -0,0 +1,86 @@
+import { regexRule } from './helpers.js';
+
+// Rules that apply to every language.
+
+const EM_DASH = '\u2014';
+const EN_DASH = '\u2013';
+
+function lineBefore(text, index) {
+  const start = text.lastIndexOf('\n', index - 1) + 1;
+  return text.slice(start, index);
+}
+
+function lineAfter(text, index) {
+  const end = text.indexOf('\n', index);
+  return text.slice(index, end === -1 ? text.length : end);
+}
+
+export default [
+  regexRule({
+    id: 'em-dash',
+    lang: '*',
+    category: 'punctuation',
+    severity: 'warning',
+    message: 'Em dash. The most recognizable AI writing tell.',
+    patterns: [new RegExp(`[ \\t]*${EM_DASH}[ \\t]*`, 'g')],
+    fix(m, text, ctx) {
+      // Leave dialogue dashes and list markers at the start of a line alone, and trailing dashes too.
+      if (lineBefore(text, m.index).trim() === '') return null;
+      if (lineAfter(text, m.index + m[0].length).trim() === '') return null;
+      return ctx.options.dashReplacement;
+    },
+  }),
+  regexRule({
+    id: 'spaced-en-dash',
+    lang: '*',
+    category: 'punctuation',
+    severity: 'warning',
+    message: 'Spaced en dash used as an em dash.',
+    // Only " - " style usage with the en dash. Ranges like 2020-2024 written with an en dash are fine.
+    patterns: [new RegExp(`(?<=\\S)[ \\t]+${EN_DASH}[ \\t]+(?=\\S)`, 'g')],
+    fix(m, text, ctx) {
+      if (lineBefore(text, m.index).trim() === '') return null;
+      return ctx.options.dashReplacement;
+    },
+  }),
+  regexRule({
+    id: 'emoji-bullet',
+    lang: '*',
+    category: 'formatting',
+    severity: 'warning',
+    message: 'Emoji used as a bullet or heading decoration.',
+    patterns: [/(?<=^[ \t]*(?:[-*+][ \t]+|\d+[.)][ \t]+|#{1,6}[ \t]+)?)\p{Extended_Pictographic}️?[ \t]*/gmu],
+    fix(m, text) {
+      // Only autofix when the emoji sits right after a list marker or heading marker.
+      const before = lineBefore(text, m.index);
+      return /^[ \t]*(?:[-*+]|\d+[.)]|#{1,6})[ \t]+$/.test(before) ? '' : null;
+    },
+  }),
+  {
+    id: 'bold-label-list',
+    lang: '*',
+    category: 'formatting',
+    severity: 'info',
+    message: 'Three or more list items in a row that start with a **Bold label:**. Classic chatbot formatting.',
+    *find(text) {
+      const lines = text.split('\n');
+      const re = /^([ \t]*(?:[-*+]|\d+[.)])[ \t]+(?:\p{Extended_Pictographic}️?[ \t]*)?)(\*\*[^*\n]{1,60}?\*\*)/u;
+      let offset = 0;
+      let runLength = 0;
+      let first = null;
+      for (let i = 0; i <= lines.length; i++) {
+        const line = lines[i];
+        const m = line === undefined ? null : re.exec(line);
+        if (m) {
+          // Point at the bold label of the first item only, so other rules on that line still report.
+          if (runLength === 0) first = { index: offset + m[1].length, length: m[2].length, quote: m[2] };
+          runLength++;
+        } else {
+          if (runLength >= 3) yield first;
+          runLength = 0;
+        }
+        if (line !== undefined) offset += line.length + 1;
+      }
+    },
+  },
+];