patina-cli 3.11.0

package/tests/quality/benchmark.mjs

@@ -0,0 +1,306 @@
+#!/usr/bin/env node
+// Quality benchmark for the deterministic stylometry layer.
+//
+// Iterates every fixture under tests/fixtures/suspect-zones/{lang}/{class}/*.md,
+// runs the in-tree analyzer (no LLM), and compares the predicted hot/cold
+// decision against the fixture's expected_hot label. Emits a per-language
+// confusion matrix + accuracy and writes the full per-fixture log to
+// tests/quality/results.json.
+//
+// Usage: node tests/quality/benchmark.mjs [--quiet]
+
+import { readFileSync, readdirSync, writeFileSync, statSync } from 'node:fs';
+import { resolve, dirname } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import yaml from 'js-yaml';
+
+import { analyzeText } from '../../src/features/index.js';
+import { loadLexicon } from '../../src/features/lexicon.js';
+
+const __dirname = dirname(fileURLToPath(import.meta.url));
+const REPO_ROOT = resolve(__dirname, '../..');
+const FIXTURES_ROOT = resolve(REPO_ROOT, 'tests/fixtures/suspect-zones');
+const RESULTS_PATH = resolve(__dirname, 'results.json');
+const EXPECTED_RANGES_PATH = resolve(FIXTURES_ROOT, 'expected-ranges.json');
+const FIXTURE_SCHEMA_VERSION = 1;
+
+const FRONTMATTER_RE = /^---\n([\s\S]*?)\n---\s*\n([\s\S]*)$/;
+
+function parseFixture(path) {
+  const raw = readFileSync(path, 'utf8');
+  const m = raw.match(FRONTMATTER_RE);
+  if (!m) throw new Error(`Missing frontmatter: ${path}`);
+  return { meta: yaml.load(m[1]), body: m[2].trim(), path };
+}
+
+function listFixtures() {
+  const out = [];
+  for (const lang of readdirSync(FIXTURES_ROOT)) {
+    const langDir = resolve(FIXTURES_ROOT, lang);
+    if (!statSync(langDir).isDirectory()) continue;
+    for (const cls of readdirSync(langDir)) {
+      const clsDir = resolve(langDir, cls);
+      if (!statSync(clsDir).isDirectory()) continue;
+      for (const file of readdirSync(clsDir)) {
+        if (!file.endsWith('.md')) continue;
+        out.push(resolve(clsDir, file));
+      }
+    }
+  }
+  return out.sort();
+}
+
+function loadExpectedRanges() {
+  const raw = readFileSync(EXPECTED_RANGES_PATH, 'utf8');
+  const ranges = JSON.parse(raw);
+  if (ranges?.schemaVersion !== 1 || !ranges?.metrics || typeof ranges.metrics !== 'object') {
+    throw new Error(`${EXPECTED_RANGES_PATH}: expected schemaVersion=1 and metrics object`);
+  }
+  return ranges.metrics;
+}
+
+function emptyMetrics() {
+  return { tp: 0, fp: 0, fn: 0, tn: 0, total: 0 };
+}
+
+function updateMetrics(m, predicted, expected) {
+  m.total++;
+  if (predicted && expected) m.tp++;
+  else if (predicted && !expected) m.fp++;
+  else if (!predicted && expected) m.fn++;
+  else m.tn++;
+}
+
+function summarize(m) {
+  const accuracy = m.total ? (m.tp + m.tn) / m.total : 0;
+  const precision = m.tp + m.fp ? m.tp / (m.tp + m.fp) : 0;
+  const recall = m.tp + m.fn ? m.tp / (m.tp + m.fn) : 0;
+  const f1 = precision + recall ? (2 * precision * recall) / (precision + recall) : 0;
+  return {
+    ...m,
+    accuracy: round(accuracy),
+    precision: round(precision),
+    recall: round(recall),
+    f1: round(f1),
+    n: m.total,
+    ci_low: round(wilsonInterval(m.tp + m.tn, m.total).low),
+    ci_high: round(wilsonInterval(m.tp + m.tn, m.total).high),
+    confidence_method: 'Wilson score interval, 95%',
+  };
+}
+
+function round(n, digits = 3) {
+  return Math.round(n * 10 ** digits) / 10 ** digits;
+}
+
+function wilsonInterval(successes, n, z = 1.959963984540054) {
+  if (!n) return { low: 0, high: 0 };
+  const phat = successes / n;
+  const denom = 1 + (z ** 2) / n;
+  const center = (phat + (z ** 2) / (2 * n)) / denom;
+  const margin = (z * Math.sqrt((phat * (1 - phat) + (z ** 2) / (4 * n)) / n)) / denom;
+  return {
+    low: Math.max(0, center - margin),
+    high: Math.min(1, center + margin),
+  };
+}
+
+function detectorHot(result) {
+  return {
+    burstiness: result.paragraphs.some((p) => p.burstiness?.band === 'low'),
+    mattr: result.paragraphs.some((p) => p.mattr?.band === 'low'),
+    lexicon: result.paragraphs.some((p) => p.lexicon?.hot),
+  };
+}
+
+function emptyDetectorMetrics() {
+  return {
+    burstiness: emptyMetrics(),
+    mattr: emptyMetrics(),
+    lexicon: emptyMetrics(),
+  };
+}
+
+function validateExpectedMetrics(path, expected = {}, observed = {}) {
+  const failures = [];
+  if (expected.cv_band && observed.cv_band !== expected.cv_band) {
+    failures.push(`cv_band expected ${expected.cv_band}, got ${observed.cv_band}`);
+  }
+  if (expected.mattr_band && observed.mattr_band !== expected.mattr_band) {
+    failures.push(`mattr_band expected ${expected.mattr_band}, got ${observed.mattr_band}`);
+  }
+  if (typeof expected.lexicon_density_min === 'number' && observed.lexicon_density < expected.lexicon_density_min) {
+    failures.push(`lexicon_density expected >= ${expected.lexicon_density_min}, got ${observed.lexicon_density}`);
+  }
+  if (typeof expected.lexicon_density_max === 'number' && observed.lexicon_density > expected.lexicon_density_max) {
+    failures.push(`lexicon_density expected <= ${expected.lexicon_density_max}, got ${observed.lexicon_density}`);
+  }
+  if (Array.isArray(expected.cv_range) && !inRange(observed.cv, expected.cv_range)) {
+    failures.push(`cv expected ${formatRange(expected.cv_range)}, got ${observed.cv}`);
+  }
+  if (Array.isArray(expected.mattr_range) && !inRange(observed.mattr, expected.mattr_range)) {
+    failures.push(`mattr expected ${formatRange(expected.mattr_range)}, got ${observed.mattr}`);
+  }
+  if (Array.isArray(expected.lexicon_density_range) && !inRange(observed.lexicon_density, expected.lexicon_density_range)) {
+    failures.push(`lexicon_density expected ${formatRange(expected.lexicon_density_range)}, got ${observed.lexicon_density}`);
+  }
+  if (expected.detectors) {
+    for (const [name, expectedHot] of Object.entries(expected.detectors)) {
+      if (observed.detectors?.[name] !== expectedHot) {
+        failures.push(`detector.${name} expected ${expectedHot}, got ${observed.detectors?.[name]}`);
+      }
+    }
+  }
+  if (typeof expected.predicted_hot === 'boolean' && observed.predicted_hot !== expected.predicted_hot) {
+    failures.push(`predicted_hot expected ${expected.predicted_hot}, got ${observed.predicted_hot}`);
+  }
+  if (failures.length) {
+    throw new Error(`${path}: expected_metrics regression failed: ${failures.join('; ')}`);
+  }
+}
+
+function inRange(value, [min, max]) {
+  return typeof value === 'number' && value >= min && value <= max;
+}
+
+function formatRange([min, max]) {
+  return `[${min}, ${max}]`;
+}
+
+function main() {
+  const quiet = process.argv.includes('--quiet');
+  const fixtures = listFixtures();
+  const expectedRanges = loadExpectedRanges();
+
+  if (fixtures.length === 0) {
+    console.error('No fixtures found under', FIXTURES_ROOT);
+    process.exit(2);
+  }
+
+  const lexicons = {};
+  const perLanguage = {};
+  const byDetector = {};
+  const fixtureLog = [];
+
+  for (const path of fixtures) {
+    const { meta, body } = parseFixture(path);
+    const lang = meta.language;
+    if (typeof meta.expected_hot !== 'boolean') {
+      throw new Error(
+        `${path}: \`expected_hot\` must be a literal boolean (got ${typeof meta.expected_hot}: ${JSON.stringify(meta.expected_hot)})`
+      );
+    }
+    if (!lexicons[lang]) lexicons[lang] = loadLexicon(lang, REPO_ROOT);
+    if (!perLanguage[lang]) perLanguage[lang] = emptyMetrics();
+    if (!byDetector[lang]) byDetector[lang] = emptyDetectorMetrics();
+
+    const result = analyzeText(body, {
+      lang,
+      lexicon: lexicons[lang],
+    });
+    const predicted = result.hot;
+    const expected = meta.expected_hot;
+    updateMetrics(perLanguage[lang], predicted, expected);
+    const detectors = detectorHot(result);
+    for (const [name, hot] of Object.entries(detectors)) {
+      updateMetrics(byDetector[lang][name], hot, expected);
+    }
+
+    const p = result.paragraphs[0] || {};
+    const observed = {
+      cv: round(p.burstiness?.cv ?? 0),
+      cv_band: p.burstiness?.band,
+      mattr: round(p.mattr?.value ?? 0),
+      mattr_band: p.mattr?.band,
+      lexicon_density: round(p.lexicon?.density ?? 0),
+      lexicon_hits: p.lexicon?.hits ?? [],
+    };
+    const pinned = expectedRanges[meta.fixture_id];
+    if (!pinned) {
+      throw new Error(
+        `${path}: missing benchmark regression range. Run node scripts/update-benchmark-ranges.mjs after reviewing the fixture.`
+      );
+    }
+    observed.detectors = detectors;
+    observed.predicted_hot = predicted;
+    validateExpectedMetrics(path, pinned, observed);
+    if (meta.expected_metrics) validateExpectedMetrics(path, meta.expected_metrics, observed);
+    fixtureLog.push({
+      fixture_id: meta.fixture_id,
+      lang,
+      class: meta.class,
+      expected_hot: expected,
+      predicted_hot: predicted,
+      correct: predicted === expected,
+      detectors,
+      ...observed,
+      expected_metrics: meta.expected_metrics ?? null,
+    });
+  }
+
+  const summary = {};
+  let totalCorrect = 0;
+  let totalCount = 0;
+  for (const [lang, m] of Object.entries(perLanguage)) {
+    summary[lang] = summarize(m);
+    summary[lang].byDetector = Object.fromEntries(
+      Object.entries(byDetector[lang]).map(([name, metrics]) => [name, summarize(metrics)])
+    );
+    totalCorrect += m.tp + m.tn;
+    totalCount += m.total;
+  }
+  const overallAccuracy = totalCount ? totalCorrect / totalCount : 0;
+  const overallCi = wilsonInterval(totalCorrect, totalCount);
+
+  const results = {
+    schemaVersion: 2,
+    fixtureSchemaVersion: FIXTURE_SCHEMA_VERSION,
+    nodeVersion: process.version,
+    generatedAt: new Date().toISOString(),
+    fixtureCount: fixtureLog.length,
+    overallAccuracy: round(overallAccuracy),
+    overall: {
+      accuracy: round(overallAccuracy),
+      n: totalCount,
+      ci_low: round(overallCi.low),
+      ci_high: round(overallCi.high),
+      confidence_method: 'Wilson score interval, 95%',
+    },
+    perLanguage: summary,
+    fixtures: fixtureLog,
+  };
+
+  writeFileSync(RESULTS_PATH, JSON.stringify(results, null, 2) + '\n');
+
+  const wrong = fixtureLog.filter((f) => !f.correct);
+
+  if (!quiet) {
+    console.log(`# Quality benchmark — ${fixtureLog.length} fixtures`);
+    console.log(`Overall accuracy: ${(overallAccuracy * 100).toFixed(1)}%`);
+    console.log();
+    console.log('| lang | n | accuracy | precision | recall | f1 | TP | FP | FN | TN |');
+    console.log('|------|---|----------|-----------|--------|----|----|----|----|----|');
+    for (const [lang, s] of Object.entries(summary)) {
+      console.log(
+        `| ${lang} | ${s.total} | ${(s.accuracy * 100).toFixed(1)}% | ${(s.precision * 100).toFixed(1)}% | ${(s.recall * 100).toFixed(1)}% | ${s.f1.toFixed(2)} | ${s.tp} | ${s.fp} | ${s.fn} | ${s.tn} |`
+      );
+    }
+    console.log();
+    if (wrong.length > 0) {
+      console.log(`Misclassified (${wrong.length}):`);
+      for (const f of wrong) {
+        console.log(
+          `  ${f.fixture_id} (${f.class}) → predicted=${f.predicted_hot}, expected=${f.expected_hot} | cv=${f.cv} ${f.cv_band}, mattr=${f.mattr} ${f.mattr_band}, lex=${f.lexicon_density}/1000`
+        );
+      }
+    } else {
+      console.log('All fixtures classified correctly.');
+    }
+    console.log(`\nFull log: ${RESULTS_PATH}`);
+  }
+
+  // Non-zero exit on any misclassification so CI catches regressions even in --quiet mode.
+  if (wrong.length > 0) process.exitCode = 1;
+}
+
+main();

package/tests/quality/detectors.manual.example.json

@@ -0,0 +1,31 @@
+{
+  "schemaVersion": 1,
+  "runId": "manual-example-2026-05-20",
+  "collectedAt": "2026-05-20",
+  "note": "Example only. Replace with manually collected third-party detector labels before drawing conclusions.",
+  "detectors": [
+    {
+      "id": "example-third-party",
+      "name": "Example Third-Party Detector",
+      "kind": "manual-third-party",
+      "scoreScale": "0..1",
+      "threshold": 0.5
+    }
+  ],
+  "results": [
+    {
+      "fixture_id": "en-ai-01",
+      "detector": "example-third-party",
+      "label": "hot",
+      "score": 0.87,
+      "notes": "Example row; do not treat as real vendor evidence."
+    },
+    {
+      "fixture_id": "en-nat-01",
+      "detector": "example-third-party",
+      "label": "cold",
+      "score": 0.12,
+      "notes": "Example row; do not treat as real vendor evidence."
+    }
+  ]
+}

package/tests/quality/dogfood.mjs

@@ -0,0 +1,44 @@
+#!/usr/bin/env node
+// Deterministic dogfood guard for public docs.
+//
+// This deliberately avoids live LLM calls in CI. It uses patina's in-tree
+// stylometry/lexicon analyzer and reports the percentage of prose paragraphs
+// that trip a hot signal. The threshold is a regression guard, not an
+// authorship verdict.
+
+import { readFileSync } from 'node:fs';
+import { dirname, resolve } from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+import { scoreText } from '../../scripts/prose-score.mjs';
+
+const __dirname = dirname(fileURLToPath(import.meta.url));
+const REPO_ROOT = resolve(__dirname, '../..');
+const THRESHOLD = 30;
+const TARGETS = [
+  { file: 'README.md', lang: 'en' },
+  { file: 'README_KR.md', lang: 'ko' },
+  { file: 'README_ZH.md', lang: 'zh' },
+  { file: 'README_JA.md', lang: 'ja' },
+  { file: 'docs/FAQ.md', lang: 'en' },
+  { file: 'SKILL.md', lang: 'ko' },
+];
+
+function scoreFile({ file, lang }) {
+  const raw = readFileSync(resolve(REPO_ROOT, file), 'utf8');
+  return scoreText(raw, { file, lang, gate: THRESHOLD, repoRoot: REPO_ROOT });
+}
+
+const rows = TARGETS.map(scoreFile);
+console.log('# Dogfood docs score');
+console.log('| file | lang | paragraphs | hot | score | threshold |');
+console.log('|---|---|---:|---:|---:|---:|');
+for (const r of rows) {
+  console.log(`| ${r.file} | ${r.lang} | ${r.paragraphCount} | ${r.hotCount} | ${r.score.toFixed(1)} | ${THRESHOLD} |`);
+}
+
+const failures = rows.filter((r) => r.score > THRESHOLD);
+if (failures.length) {
+  console.error(`\nDogfood score exceeded ${THRESHOLD}: ${failures.map((f) => `${f.file}=${f.score.toFixed(1)}`).join(', ')}`);
+  process.exitCode = 1;
+}