patina-cli 3.11.0 → 4.0.0

package/scripts/benchmark-report.mjs

@@ -43,14 +43,17 @@ function readResults() {
 
 function validateResultsSchema(results) {
   const missing = [];
-  if (results?.schemaVersion !== 2) missing.push('schemaVersion=2');
+  if (results?.schemaVersion !== 3) missing.push('schemaVersion=3');
   if (typeof results?.fixtureSchemaVersion !== 'number') missing.push('fixtureSchemaVersion');
   if (typeof results?.nodeVersion !== 'string') missing.push('nodeVersion');
   if (typeof results?.overall?.ci_low !== 'number') missing.push('overall.ci_low');
   if (typeof results?.overall?.ci_high !== 'number') missing.push('overall.ci_high');
   if (typeof results?.overall?.n !== 'number') missing.push('overall.n');
+  if (!isNumberOrNull(results?.ranking?.overall?.roc_auc)) missing.push('ranking.overall.roc_auc');
+  if (!isNumberOrNull(results?.ranking?.overall?.pr_auc)) missing.push('ranking.overall.pr_auc');
+  if (!results?.ranking?.overall?.bestF1) missing.push('ranking.overall.bestF1');
   for (const [lang, summary] of Object.entries(results?.perLanguage || {})) {
-    for (const detector of ['burstiness', 'mattr', 'lexicon']) {
+    for (const detector of ['burstiness', 'koDiagnostics', 'mattr', 'lexicon']) {
       if (!summary.byDetector?.[detector]) missing.push(`perLanguage.${lang}.byDetector.${detector}`);
     }
   }
@@ -59,6 +62,10 @@ function validateResultsSchema(results) {
   }
 }
 
+function isNumberOrNull(value) {
+  return value === null || typeof value === 'number';
+}
+
 function pct(value) {
   return `${((value ?? 0) * 100).toFixed(1)}%`;
 }
@@ -67,6 +74,14 @@ function num(value, digits = 3) {
   return Number(value ?? 0).toFixed(digits).replace(/\.0+$/, '').replace(/(\.\d*?)0+$/, '$1');
 }
 
+function optionalPct(value) {
+  return value === null || value === undefined ? '—' : pct(value);
+}
+
+function optionalNum(value, digits = 3) {
+  return value === null || value === undefined ? '—' : num(value, digits);
+}
+
 function bool(value) {
   return value ? 'hot' : 'cold';
 }
@@ -103,6 +118,18 @@ function detectorRows(perLanguage = {}) {
   return rows;
 }
 
+function rankingRows(ranking = {}) {
+  const rows = [];
+  if (ranking.overall) rows.push(['overall', ranking.overall]);
+  for (const [lang, summary] of Object.entries(ranking.perLanguage || {}).sort(([a], [b]) => a.localeCompare(b))) {
+    rows.push([lang, summary]);
+  }
+  return rows.map(([scope, summary]) => {
+    const best = summary.bestF1 || {};
+    return `| ${cell(scope)} | ${summary.n} | ${summary.positives} | ${summary.negatives} | ${optionalNum(summary.roc_auc)} | ${optionalNum(summary.pr_auc)} | ${optionalNum(best.threshold)} | ${optionalPct(best.precision)} | ${optionalPct(best.recall)} | ${optionalNum(best.f1, 2)} | ${optionalPct(best.accuracy)} |`;
+  });
+}
+
 function classRows(fixtures = []) {
   const counts = new Map();
   for (const f of fixtures) {
@@ -130,7 +157,10 @@ function sampleSizeSummary(fixtures = []) {
 function fixtureRows(fixtures = []) {
   return fixtures.map((f) => {
     const hits = cell((f.lexicon_hits || []).slice(0, 4).join(', '));
-    return `| ${cell(f.fixture_id)} | ${cell(f.lang)} | ${cell(f.class)} | ${bool(f.expected_hot)} | ${bool(f.predicted_hot)} | ${resultMark(f.correct)} | ${num(f.cv)} ${cell(f.cv_band)} | ${num(f.mattr)} ${cell(f.mattr_band)} | ${num(f.lexicon_density)} | ${hits} |`;
+    const koDiag = f.ko_diagnostics_hot
+      ? `hot: ${(f.ko_diagnostics_reasons || []).join(', ')}`
+      : 'cold';
+    return `| ${cell(f.fixture_id)} | ${cell(f.lang)} | ${cell(f.class)} | ${bool(f.expected_hot)} | ${bool(f.predicted_hot)} | ${resultMark(f.correct)} | ${num(f.signal_score)} | ${num(f.cv)} ${cell(f.cv_band)} | ${num(f.mattr)} ${cell(f.mattr_band)} | ${num(f.lexicon_density)} | ${cell(koDiag)} | ${hits} |`;
   });
 }
 
@@ -180,7 +210,7 @@ This is the latest checked-in report for patina's deterministic suspect-zone ben
 - Regression ranges: \`tests/fixtures/suspect-zones/expected-ranges.json\` (refresh with \`npm run benchmark:ranges\`)
 - Reproduce: \`npm run benchmark:report\`
 - Raw JSON: [latest.json](latest.json)
-- Detector comparison harness: [detector-comparison.md](detector-comparison.md)
+- Detector comparison protocol: [detector-comparison.md](detector-comparison.md)
 - 2025+ re-baseline plan: [docs/research/2025-rebaseline-plan.md](../research/2025-rebaseline-plan.md)
 
 ## Language breakdown
@@ -195,6 +225,16 @@ ${languageRows(results.perLanguage).join('\n')}
 |---|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|
 ${detectorRows(results.perLanguage).join('\n')}
 
+## Ranking diagnostics
+
+Signal-score ranking shows whether the diagnostic \`signal_score\` separates hot
+fixtures from natural fixtures before any threshold is chosen. It is computed
+only on the checked-in fixture corpus and is not a broader model-era claim.
+
+| scope | fixtures | positives | negatives | ROC-AUC | PR-AUC | best threshold | precision | recall | best F1 | accuracy |
+|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|
+${rankingRows(results.ranking).join('\n')}
+
 ## Sample sizes
 
 | lang | class | fixtures |
@@ -207,14 +247,15 @@ ${misclassificationSection(results.fixtures)}
 
 ## Fixture log
 
-| fixture | lang | class | expected | predicted | ok | CV band | MATTR band | lexicon/1k | sample lexicon hits |
-|---|---|---|---|---|---:|---:|---:|---:|---|
+| fixture | lang | class | expected | predicted | ok | signal | CV band | MATTR band | lexicon/1k | KO diagnostic | sample lexicon hits |
+|---|---|---|---|---|---:|---:|---:|---:|---:|---|---|
 ${fixtureRows(results.fixtures).join('\n')}
 
 ## How to read this
 
-- **Hot** means at least one deterministic signal crossed the benchmark threshold: low burstiness CV, low MATTR, or AI-lexicon density.
+- **Hot** means at least one deterministic signal crossed the benchmark threshold: low burstiness CV, low MATTR, AI-lexicon density, or the conservative Korean diagnostic composite.
 - **Cold** means the fixture did not cross those thresholds.
+- **Signal** is the 0–100 diagnostic strength of the strongest deterministic trigger. It supports ranking diagnostics but does not replace the binary hot/cold regression gate.
 - The report is meant for regression tracking and contributor discussion, not for authorship accusation.
 - This deterministic corpus is intentionally small (${results.fixtureCount} fixtures across ${languageList}); do not treat 100% fixture accuracy as generalization to new models, genres, or edited AI text.
 - Confidence intervals use Wilson score intervals for the checked-in fixture set; external threshold sweeps and 2025+ model rebaselines are separate research follow-ups tracked in [2025+ Re-baseline Plan](../research/2025-rebaseline-plan.md).
@@ -230,7 +271,7 @@ function main() {
   if (!runBenchmarkFirst) benchmarkStatus = statusFromResults(results);
 
   const report = {
-    reportVersion: 2,
+    reportVersion: 3,
     benchmarkCommand: benchmarkCommand.join(' '),
     benchmarkStatus,
     note: 'Deterministic suspect-zone benchmark; not an authorship detector.',
@@ -249,4 +290,10 @@ function main() {
   if (benchmarkStatus !== 0) process.exitCode = benchmarkStatus;
 }
 
-main();
+const isDirectRun = process.argv[1]
+  ? resolve(process.cwd(), process.argv[1]) === fileURLToPath(import.meta.url)
+  : false;
+
+if (isDirectRun) main();
+
+export { renderMarkdown, validateResultsSchema };

package/scripts/check-release-metadata.mjs

@@ -10,8 +10,6 @@ expect(pkg.bin?.patina === 'bin/patina.js', 'package.json bin.patina must point
 expect(pkg.bin?.['patina-score'] === 'scripts/precommit-score.mjs', 'package.json bin.patina-score must point to scripts/precommit-score.mjs');
 expect(existsSync('bin/patina.js'), 'bin/patina.js must exist');
 expect(readVersionField('SKILL.md') === version, 'SKILL.md version must match package.json');
-expect(readVersionField('SKILL-MAX.md') === version, 'SKILL-MAX.md version must match package.json');
-expect(readVersionField('patina-max/SKILL.md') === version, 'patina-max/SKILL.md version must match package.json');
 expect(readVersionField('.patina.default.yaml') === version, '.patina.default.yaml version must match package.json');
 expect(readFileSync('README.md', 'utf8').includes(`version: "${version}"`), 'README.md config example version must match package.json');
 expect(new RegExp(`^## ${escapeRegex(version)} — \\d{4}-\\d{2}-\\d{2}`, 'm').test(readFileSync('CHANGELOG.md', 'utf8')), 'CHANGELOG.md must contain a release heading for package.json version');

package/scripts/detector-comparison.mjs

@@ -1,5 +1,5 @@
 #!/usr/bin/env node
-// Offline detector-comparison harness for the suspect-zone benchmark.
+// Offline detector-comparison protocol for the suspect-zone benchmark.
 //
 // Default mode compares Patina's deterministic in-tree analyzer against the
 // checked-in fixture labels. Pass --input <manual-results.json> to merge scores
@@ -42,8 +42,8 @@ function runBenchmark() {
 
 function readBenchmarkResults() {
   const results = JSON.parse(readFileSync(RESULTS_PATH, 'utf8'));
-  if (results?.schemaVersion !== 2 || !Array.isArray(results?.fixtures)) {
-    throw new Error(`${relative(REPO_ROOT, RESULTS_PATH)} is not a benchmark schema v2 result`);
+  if (results?.schemaVersion !== 3 || !Array.isArray(results?.fixtures)) {
+    throw new Error(`${relative(REPO_ROOT, RESULTS_PATH)} is not a benchmark schema v3 result`);
   }
   return results;
 }
@@ -121,7 +121,7 @@ function builtInDetector(results) {
         name: 'Patina deterministic suspect-zone analyzer',
         kind: 'in-tree',
         mode: 'offline',
-        threshold: 'burstiness low OR MATTR low OR lexicon density > threshold',
+        threshold: 'burstiness low OR MATTR low OR lexicon density > threshold with min hits OR koDiagnostics hot',
       },
     ],
     rows: results.fixtures.map((fixture) => ({
@@ -205,9 +205,9 @@ function fixtureRows(rows) {
 }
 
 function renderMarkdown(report) {
-  return `# Detector Comparison Harness
+  return `# Detector Comparison Protocol
 
-This report is generated offline from the checked-in suspect-zone fixtures. It is a comparison harness, not a vendor ranking claim.
+This report is generated offline from the checked-in suspect-zone fixtures. It is a comparison protocol, not a vendor ranking claim.
 
 ## Current run
 
@@ -251,7 +251,7 @@ function main() {
     generatedAt: new Date().toISOString(),
     fixtureCount: results.fixtureCount,
     benchmarkGeneratedAt: results.generatedAt,
-    note: 'Offline comparison harness. Built-in Patina row uses deterministic suspect-zone analyzer; third-party rows are manual opt-in only.',
+    note: 'Offline comparison protocol. Built-in Patina row uses deterministic suspect-zone analyzer; third-party rows are manual opt-in only.',
     manualInput: manual ? relative(REPO_ROOT, manual.path) : null,
     detectors,
     summaries: computed.summaries,

package/scripts/generate-playground-data.mjs

@@ -0,0 +1,77 @@
+#!/usr/bin/env node
+// Generate the browser-safe lexicon bundle for the static playground.
+// Keep this dependency-free so Vercel can serve committed assets without a build.
+
+import { readFileSync, writeFileSync } from 'node:fs';
+import { dirname, resolve } from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+const __dirname = dirname(fileURLToPath(import.meta.url));
+const REPO_ROOT = resolve(__dirname, '..');
+const LANGS = ['en', 'ko', 'zh', 'ja'];
+
+function parseFrontmatter(raw) {
+  if (!raw.startsWith('---')) return { meta: {}, body: raw };
+  const end = raw.indexOf('\n---', 3);
+  if (end === -1) return { meta: {}, body: raw };
+  const yaml = raw.slice(3, end).trim();
+  const meta = {};
+  for (const line of yaml.split('\n')) {
+    const match = line.match(/^([A-Za-z0-9_-]+):\s*(.*)$/u);
+    if (match) meta[match[1]] = match[2].trim();
+  }
+  return { meta, body: raw.slice(end + 4) };
+}
+
+function parseLexiconBody(body) {
+  const strict = [];
+  const phrases = [];
+  let mode = null;
+  for (const rawLine of body.split('\n')) {
+    const line = rawLine.trim();
+    if (line.startsWith('## ')) {
+      const heading = line.toLowerCase();
+      if (heading.includes('strict matches')) mode = 'strict';
+      else if (heading.includes('multi-word phrases')) mode = 'phrases';
+      else mode = null;
+      continue;
+    }
+    if (!mode || !line.startsWith('- ')) continue;
+    const entry = line.slice(2).trim().normalize('NFC');
+    if (entry && !entry.startsWith('_')) (mode === 'strict' ? strict : phrases).push(entry);
+  }
+  return { strict, phrases };
+}
+
+const lexicons = {};
+for (const lang of LANGS) {
+  const source = `lexicon/ai-${lang}.md`;
+  const raw = readFileSync(resolve(REPO_ROOT, source), 'utf8');
+  const { meta, body } = parseFrontmatter(raw);
+  const parsed = parseLexiconBody(body);
+  lexicons[lang] = {
+    lang,
+    source,
+    version: meta.version || null,
+    strict: parsed.strict,
+    phrases: parsed.phrases,
+  };
+}
+
+const output = `// Generated by scripts/generate-playground-data.mjs. Do not edit by hand.\n` +
+  `// Source: lexicon/ai-{en,ko,zh,ja}.md\n\n` +
+  `export const PLAYGROUND_LEXICONS = ${JSON.stringify(lexicons, null, 2)};\n`;
+
+const target = resolve(REPO_ROOT, 'playground/data/lexicons.js');
+if (process.argv.includes('--check')) {
+  const current = readFileSync(target, 'utf8');
+  if (current !== output) {
+    console.error('playground/data/lexicons.js is stale; run npm run playground:data');
+    process.exitCode = 1;
+  } else {
+    console.log('playground/data/lexicons.js is up to date');
+  }
+} else {
+  writeFileSync(target, output, 'utf8');
+  console.log('Wrote playground/data/lexicons.js');
+}