patina-cli 3.11.0

package/src/features/index.js

@@ -0,0 +1,96 @@
+// Top-level analyzer: runs the deterministic stylometry + lexicon signals
+// described in core/stylometry.md and returns a per-paragraph result. This
+// is the in-tree port of the algorithm previously delegated to the LLM via
+// SKILL.md Step 4.6/4.7. It does not call any LLM.
+
+import { splitParagraphs, splitSentences, tokenize } from './segment.js';
+import {
+  burstinessCV,
+  mattr,
+  classifyBurstiness,
+  classifyMattr,
+  DEFAULT_BURSTINESS_BANDS,
+  DEFAULT_MATTR_BANDS,
+  DEFAULT_MATTR_WINDOW,
+} from './stylometry.js';
+import { loadLexicon, computeDensity, DEFAULT_LEXICON_DENSITY_THRESHOLD } from './lexicon.js';
+
+export function analyzeText(text, opts = {}) {
+  const {
+    lang = 'en',
+    repoRoot,
+    burstinessBands = DEFAULT_BURSTINESS_BANDS,
+    mattrBands = DEFAULT_MATTR_BANDS,
+    mattrWindow = DEFAULT_MATTR_WINDOW,
+    lexiconDensityThreshold = DEFAULT_LEXICON_DENSITY_THRESHOLD,
+    lexicon: providedLexicon,
+  } = opts;
+
+  // Normalize to NFC at the boundary so downstream tokenization and lexicon
+  // comparison see canonical form. Mixed NFC/NFD inputs (e.g. "café" composed
+  // vs decomposed) would otherwise yield different MATTR/lexicon hits.
+  const normalized = text ? text.normalize('NFC') : '';
+  const paragraphs = splitParagraphs(normalized);
+  const lexicon =
+    providedLexicon ??
+    (repoRoot ? loadLexicon(lang, repoRoot) : { strict: [], phrases: [] });
+
+  // §8 skip conditions are advisory only — production callers (SKILL.md 4.6/4.7)
+  // can suppress meta-block emission, but the benchmark wants raw signals on
+  // single-paragraph fixtures so we compute them unconditionally.
+  const totalSentences = paragraphs.reduce(
+    (n, p) => n + splitSentences(p).length,
+    0
+  );
+  const skipReason =
+    paragraphs.length <= 2 ? 'paragraphs<=2' :
+    totalSentences <= 2 ? 'sentences<=2' :
+    null;
+
+  const analyzed = paragraphs.map((paragraph, idx) => {
+    const sentences = splitSentences(paragraph);
+    const sentenceTokens = sentences.map((sentence) => tokenize(sentence, { lang }));
+    const sentenceTokenCounts = sentenceTokens.map((t) => t.length);
+    const allTokens = sentenceTokens.flat();
+
+    const cv = burstinessCV(sentenceTokenCounts);
+    const cvBand = classifyBurstiness(cv, burstinessBands);
+    const mattrValue = mattr(allTokens, mattrWindow);
+    const mattrBand = classifyMattr(mattrValue, mattrBands);
+    const lex = computeDensity(paragraph, allTokens, lexicon);
+
+    const lexiconHot = lex.density > lexiconDensityThreshold;
+    const hot =
+      cvBand === 'low' || mattrBand === 'low' || lexiconHot;
+
+    return {
+      id: `P${idx + 1}`,
+      sentenceCount: sentences.length,
+      tokenCount: allTokens.length,
+      burstiness: { cv, band: cvBand },
+      mattr: { value: mattrValue, band: mattrBand },
+      lexicon: { ...lex, hot: lexiconHot },
+      hot,
+    };
+  });
+
+  return {
+    lang,
+    skipped: Boolean(skipReason),
+    skipReason,
+    paragraphs: analyzed,
+    hot: analyzed.some((p) => p.hot),
+  };
+}
+
+export {
+  splitParagraphs,
+  splitSentences,
+  tokenize,
+  burstinessCV,
+  mattr,
+  classifyBurstiness,
+  classifyMattr,
+  loadLexicon,
+  computeDensity,
+};

package/src/features/lexicon.js

@@ -0,0 +1,90 @@
+// AI-lexicon loading and density per core/stylometry.md §16.
+// Loads the markdown lexicon at lexicon/ai-{lang}.md and computes
+// hits-per-1000-tokens density. Custom lexicons under custom/lexicon/
+// take precedence when present.
+
+import { readFileSync, existsSync } from 'node:fs';
+import { resolve } from 'node:path';
+
+export const DEFAULT_LEXICON_DENSITY_THRESHOLD = 2.0;
+
+// Parses the two well-known sections out of a lexicon markdown file.
+// Returns { strict: string[], phrases: string[] }.
+function parseLexiconBody(body) {
+  const strict = [];
+  const phrases = [];
+  let mode = null;
+  for (const rawLine of body.split('\n')) {
+    const line = rawLine.trim();
+    if (line.startsWith('## ')) {
+      const heading = line.toLowerCase();
+      if (heading.includes('strict matches')) mode = 'strict';
+      else if (heading.includes('multi-word phrases')) mode = 'phrases';
+      else mode = null;
+      continue;
+    }
+    if (mode && line.startsWith('- ')) {
+      // Normalize to NFC so visually identical entries don't fail to match
+      // tokens that arrive in a different normalization form.
+      const entry = line.slice(2).trim().normalize('NFC');
+      if (entry) (mode === 'strict' ? strict : phrases).push(entry);
+    }
+  }
+  return { strict, phrases };
+}
+
+export function loadLexicon(lang, repoRoot) {
+  const candidates = [
+    resolve(repoRoot, 'custom', 'lexicon', `ai-${lang}.md`),
+    resolve(repoRoot, 'lexicon', `ai-${lang}.md`),
+  ];
+  for (const path of candidates) {
+    if (existsSync(path)) {
+      const raw = readFileSync(path, 'utf8');
+      const body = raw.replace(/^---[\s\S]*?---\s*/, '');
+      return { lang, path, ...parseLexiconBody(body) };
+    }
+  }
+  return { lang, path: null, strict: [], phrases: [] };
+}
+
+// Phrases may include `~` as a wildcard standing in for up to 40 chars.
+function phraseToRegex(phrase) {
+  const escaped = phrase
+    .toLowerCase()
+    .replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+  const withWildcard = escaped.replace(/~/g, '.{0,40}');
+  return new RegExp(withWildcard);
+}
+
+// Counts paragraph-level lexicon hits. Strict entries match whole-word
+// (Unicode-aware boundaries via \p{L}\p{N}); phrases match as substrings
+// with `~` wildcard support. Each entry counts at most once per paragraph.
+export function computeDensity(paragraphText, tokens, lexicon) {
+  const lowerText = paragraphText.toLowerCase();
+  const hits = [];
+  const tokenSet = new Set(tokens.map((t) => t.toLowerCase()));
+
+  // §16: English strict entries match whole-word; Korean strict entries are
+  // approximated by substring (어절 inflection means `자리매김` should also
+  // hit `자리매김했다`, `자리매김으로`, etc.). Punctuated entries always need
+  // substring fallback because tokenization strips edge punct.
+  const koSubstring = lexicon.lang === 'ko';
+  for (const entry of lexicon.strict) {
+    const lowerEntry = entry.toLowerCase();
+    if (tokenSet.has(lowerEntry)) {
+      hits.push(entry);
+      continue;
+    }
+    const hasInternalPunct = /[^\p{L}\p{N}]/u.test(lowerEntry);
+    if ((koSubstring || hasInternalPunct) && lowerText.includes(lowerEntry)) {
+      hits.push(entry);
+    }
+  }
+  for (const phrase of lexicon.phrases) {
+    if (phraseToRegex(phrase).test(lowerText)) hits.push(phrase);
+  }
+
+  const density = tokens.length > 0 ? (hits.length / tokens.length) * 1000 : 0;
+  return { matches: hits.length, density, hits };
+}

package/src/features/segment.js

@@ -0,0 +1,49 @@
+// Paragraph / sentence / token splitting per core/stylometry.md §2 §3.
+//
+// Tokenization is intentionally simple: whitespace split + edge-punctuation
+// strip, no morphological analysis. For Chinese/Japanese prose, where normal
+// text often has no whitespace, use a deterministic character-token fallback
+// so sentence-length and lexical-diversity signals are not collapsed to
+// "one token per sentence." Sentence splitting is regex-only and accepts known
+// false splits on abbreviations / decimals (documented limit).
+
+const SENTENCE_SPLIT_RE = /[.!?]+\s+|(?<=[。！？…])|\n+/u;
+const PARAGRAPH_SPLIT_RE = /\n\s*\n/;
+// \W in Unicode-aware mode. Strips edge punctuation but keeps internal
+// hyphens / apostrophes (e.g. "don't", "좋은-도구") as a single token.
+const EDGE_PUNCT_RE = /^[^\p{L}\p{N}]+|[^\p{L}\p{N}]+$/gu;
+const CJK_TOKEN_RE = /[\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}\u30FC]|[A-Za-z0-9]+/gu;
+
+export function splitParagraphs(text) {
+  if (!text) return [];
+  return text
+    .split(PARAGRAPH_SPLIT_RE)
+    .map((p) => p.trim())
+    .filter((p) => p.length > 0);
+}
+
+export function splitSentences(paragraph) {
+  if (!paragraph) return [];
+  return paragraph
+    .split(SENTENCE_SPLIT_RE)
+    .map((s) => s.trim().replace(/[.!?。！？…]+$/u, ''))
+    .filter((s) => s.length > 0);
+}
+
+function tokenizeCjk(text) {
+  const tokens = [];
+  for (const match of text.matchAll(CJK_TOKEN_RE)) {
+    const token = match[0].replace(EDGE_PUNCT_RE, '');
+    if (token) tokens.push(token);
+  }
+  return tokens;
+}
+
+export function tokenize(text, opts = {}) {
+  if (!text) return [];
+  if (opts.lang === 'zh' || opts.lang === 'ja') return tokenizeCjk(text);
+  return text
+    .split(/\s+/)
+    .map((chunk) => chunk.replace(EDGE_PUNCT_RE, ''))
+    .filter((t) => t.length > 0);
+}

package/src/features/stylometry.js

@@ -0,0 +1,50 @@
+// Burstiness CV + MATTR per core/stylometry.md §4 §5.
+// Pure functions over token arrays; no I/O.
+
+export const DEFAULT_BURSTINESS_BANDS = { low: 0.30, high: 0.50 };
+export const DEFAULT_MATTR_BANDS = { low: 0.55, high: 0.70 };
+export const DEFAULT_MATTR_WINDOW = 50;
+
+// Coefficient of variation of sentence token counts.
+// Returns null when the paragraph has fewer than 2 sentences or mean is 0.
+export function burstinessCV(sentenceTokenCounts) {
+  if (!Array.isArray(sentenceTokenCounts) || sentenceTokenCounts.length < 2) return null;
+  const n = sentenceTokenCounts.length;
+  const mean = sentenceTokenCounts.reduce((a, b) => a + b, 0) / n;
+  if (mean === 0) return null;
+  const variance =
+    sentenceTokenCounts.reduce((acc, x) => acc + (x - mean) ** 2, 0) / n;
+  return Math.sqrt(variance) / mean;
+}
+
+// Moving Average Type-Token Ratio (window default 50).
+// Falls back to simple TTR when token count < window.
+export function mattr(tokens, window = DEFAULT_MATTR_WINDOW) {
+  if (!Array.isArray(tokens) || tokens.length === 0) return null;
+  const lower = tokens.map((t) => t.toLowerCase());
+  if (lower.length < window) {
+    return new Set(lower).size / lower.length;
+  }
+  let sum = 0;
+  let count = 0;
+  for (let i = 0; i + window <= lower.length; i++) {
+    const slice = lower.slice(i, i + window);
+    sum += new Set(slice).size / window;
+    count++;
+  }
+  return sum / count;
+}
+
+export function classifyBurstiness(cv, bands = DEFAULT_BURSTINESS_BANDS) {
+  if (cv == null) return null;
+  if (cv < bands.low) return 'low';
+  if (cv > bands.high) return 'high';
+  return 'mid';
+}
+
+export function classifyMattr(value, bands = DEFAULT_MATTR_BANDS) {
+  if (value == null) return null;
+  if (value < bands.low) return 'low';
+  if (value > bands.high) return 'high';
+  return 'mid';
+}

package/src/loader.js

@@ -0,0 +1,103 @@
+import { readFileSync, readdirSync } from 'node:fs';
+import { resolve, sep } from 'node:path';
+import yaml from 'js-yaml';
+import { validateProfileName } from './security.js';
+
+export function loadFile(path) {
+  return readFileSync(path, 'utf8');
+}
+
+export function splitFrontmatter(content) {
+  const match = content.match(/^---\s*\n([\s\S]*?)\n---\s*\n([\s\S]*)$/);
+  if (!match) {
+    return { frontmatter: null, body: content };
+  }
+  return {
+    frontmatter: yaml.load(match[1]),
+    body: match[2].trim(),
+  };
+}
+
+export function loadPatterns(repoRoot, lang, skipPatterns = []) {
+  const patternsDir = resolve(repoRoot, 'patterns');
+  const files = readdirSync(patternsDir)
+    .filter((f) => f.startsWith(`${lang}-`) && f.endsWith('.md'))
+    .filter((f) => {
+      const packName = f.replace('.md', '');
+      return !skipPatterns.includes(packName);
+    })
+    .sort();
+
+  const packs = [];
+  for (const file of files) {
+    const content = loadFile(resolve(patternsDir, file));
+    const { frontmatter, body } = splitFrontmatter(content);
+    packs.push({
+      file,
+      frontmatter,
+      body,
+      isStructure: frontmatter?.phase === 'structure',
+      isScoreOnly: frontmatter?.score_only === true,
+    });
+  }
+  return packs;
+}
+
+export function loadProfile(repoRoot, profileName) {
+  validateProfileName(profileName);
+  const profilesDir = resolve(repoRoot, 'profiles');
+  const profilePath = resolve(profilesDir, `${profileName}.md`);
+  if (!profilePath.startsWith(profilesDir + sep)) {
+    throw new Error(`Profile path escaped profiles/: ${profilePath}`);
+  }
+  const content = loadFile(profilePath);
+  return splitFrontmatter(content);
+}
+
+export function loadCoreFile(repoRoot, filename) {
+  const path = resolve(repoRoot, 'core', filename);
+  const content = loadFile(path);
+  return splitFrontmatter(content);
+}
+
+export function loadInputText(path) {
+  return readFileSync(path, 'utf8');
+}
+
+export function loadVoiceSample(path) {
+  const content = loadFile(path);
+  const paragraphs = content
+    .split(/\n\s*\n/)
+    .map((paragraph) => paragraph.trim())
+    .filter(Boolean);
+
+  if (paragraphs.length === 0) {
+    throw new Error(`Voice sample is empty: ${path}`);
+  }
+
+  const selected = paragraphs.slice(0, 3);
+  return {
+    path,
+    paragraphs: selected,
+    body: selected.join('\n\n'),
+    truncated: paragraphs.length > selected.length,
+  };
+}
+
+// Tone → backbone profile mapping (v3.10, mirrors SKILL.md Phase 1).
+// Returns the *primary* backbone profile name for a resolved tone.
+// Multi-profile tones (e.g. professional → email + formal + legal + medical)
+// expose only the primary here; secondary profiles are documented in SKILL.md
+// and respected via legal/medical fidelity-floor enforcement at Phase 5b.
+const TONE_BACKBONE = {
+  casual: 'blog',                 // primary; social is a secondary backbone
+  professional: 'email',          // primary; formal/legal/medical secondary
+  academic: 'academic',           // primary; technical secondary
+  narrative: 'narrative',
+  marketing: 'marketing',
+  instructional: 'instructional',
+};
+
+export function toneToBackboneProfile(tone) {
+  return TONE_BACKBONE[tone] || null;
+}

package/src/logger.js

@@ -0,0 +1,70 @@
+const LEVELS = {
+  debug: 10,
+  info: 20,
+  warn: 30,
+  error: 40,
+  silent: Infinity,
+};
+
+export function createLogger({
+  level = process.env.PATINA_LOG_LEVEL || 'info',
+  quiet = false,
+  json = false,
+  stream = process.stderr,
+} = {}) {
+  const threshold = quiet ? LEVELS.silent : (LEVELS[String(level).toLowerCase()] ?? LEVELS.info);
+  let progressOpen = false;
+
+  const emit = (levelName, event, fields = {}) => {
+    if (LEVELS[levelName] < threshold) return;
+    closeProgress();
+    if (json) {
+      console.error(JSON.stringify(record(levelName, event, fields)));
+      return;
+    }
+    if (fields.message) console.error(fields.message);
+  };
+
+  const progress = (event, fields = {}) => {
+    if (LEVELS.info < threshold) return;
+    if (json) {
+      console.error(JSON.stringify(record('info', event, fields)));
+      return;
+    }
+    if (!fields.message || !stream?.write) return;
+    stream.write(`\r${fields.message}`);
+    progressOpen = true;
+  };
+
+  function closeProgress() {
+    if (progressOpen && stream?.write) stream.write('\n');
+    progressOpen = false;
+  }
+
+  return {
+    debug: (event, fields) => emit('debug', event, fields),
+    info: (event, fields) => emit('info', event, fields),
+    warn: (event, fields) => emit('warn', event, fields),
+    error: (event, fields) => emit('error', event, fields),
+    progress,
+    closeProgress,
+    child(extra = {}) {
+      return createLogger({ level, quiet, json, stream, ...extra });
+    },
+  };
+}
+
+function record(level, event, fields = {}) {
+  const { message, model = null, latency_ms = null, ...rest } = fields;
+  return {
+    ts: new Date().toISOString(),
+    level,
+    event,
+    model,
+    latency_ms,
+    ...(message ? { message } : {}),
+    ...rest,
+  };
+}
+
+export const defaultLogger = createLogger();

package/src/manifest.js

@@ -0,0 +1,162 @@
+// Reproducibility manifest writer — captures enough metadata about a run
+// to reproduce it later (config hash, prompt hash, selected patterns,
+// provider/model, package version, results). Schema is versioned so
+// callers and tooling can detect breaking shape changes.
+
+import { createHash } from 'node:crypto';
+import { readFileSync, writeFileSync, mkdirSync } from 'node:fs';
+import { resolve } from 'node:path';
+
+export const MANIFEST_SCHEMA_VERSION = '2';
+
+export function hashSha256(input) {
+  if (input == null) return null;
+  const data = typeof input === 'string' ? input : JSON.stringify(input);
+  return `sha256:${createHash('sha256').update(data).digest('hex')}`;
+}
+
+// Build the manifest body. Caller passes the already-resolved run state;
+// this function is pure (no I/O) so it's easy to unit-test.
+export function buildManifest({
+  patinaVersion,
+  mode,
+  lang,
+  profile,
+  provider,
+  backend,
+  model,
+  configPath,
+  config,
+  patterns,
+  results,
+  startedAt,
+  finishedAt = new Date().toISOString(),
+  temperature = null,
+  seed = null,
+}) {
+  return {
+    manifestVersion: MANIFEST_SCHEMA_VERSION,
+    patina: patinaVersion,
+    startedAt,
+    finishedAt,
+    mode,
+    lang,
+    profile,
+    provider: provider ?? null,
+    backend: backend ?? null,
+    model: model ?? null,
+    temperature,
+    seed,
+    configPath: configPath ?? null,
+    configHash: hashSha256(config),
+    patterns: (patterns ?? []).map((p) => p.frontmatter?.pack || p.file),
+    results: results ?? [],
+  };
+}
+
+// Add one input/output pair's hash + ref to the running results array.
+// Mutates the input array for convenience.
+export function appendResult(
+  results,
+  {
+    inputPath,
+    prompt,
+    outputRef,
+    response,
+    tokensIn = null,
+    tokensOut = null,
+    temperature = null,
+    seed = null,
+    cost = null,
+    scores,
+    iterationLog,
+    calls,
+  }
+) {
+  const entry = {
+    input: inputPath,
+    promptHash: hashSha256(prompt),
+    responseHash: hashSha256(response),
+    output: outputRef,
+    tokensIn,
+    tokensOut,
+    temperature,
+    seed,
+    cost,
+  };
+  if (scores) entry.scores = scores;
+  if (iterationLog) entry.iterationLog = iterationLog;
+  if (calls) entry.calls = calls;
+  results.push(entry);
+  return results;
+}
+
+export function readManifest(path) {
+  return normalizeManifest(JSON.parse(readFileSync(path, 'utf8')));
+}
+
+export function normalizeManifest(manifest) {
+  if (!manifest || typeof manifest !== 'object') {
+    throw new Error('Manifest must be a JSON object');
+  }
+
+  const version = String(manifest.manifestVersion ?? '1');
+  if (version === MANIFEST_SCHEMA_VERSION) {
+    return {
+      ...manifest,
+      results: normalizeV2Results(manifest.results),
+    };
+  }
+
+  if (version === '1') {
+    return {
+      ...manifest,
+      manifestVersion: '1',
+      temperature: manifest.temperature ?? null,
+      seed: manifest.seed ?? null,
+      results: normalizeV1Results(manifest.results),
+    };
+  }
+
+  throw new Error(`Unsupported manifest schema version: ${version}`);
+}
+
+function normalizeV1Results(results) {
+  return (results ?? []).map((entry) => ({
+    input: entry.input ?? null,
+    promptHash: entry.promptHash ?? null,
+    responseHash: entry.responseHash ?? null,
+    output: entry.output ?? null,
+    tokensIn: entry.tokensIn ?? null,
+    tokensOut: entry.tokensOut ?? null,
+    temperature: entry.temperature ?? null,
+    seed: entry.seed ?? null,
+    cost: entry.cost ?? null,
+    ...(entry.scores ? { scores: entry.scores } : {}),
+    ...(entry.iterationLog ? { iterationLog: entry.iterationLog } : {}),
+    ...(entry.calls ? { calls: entry.calls } : {}),
+  }));
+}
+
+function normalizeV2Results(results) {
+  return (results ?? []).map((entry) => ({
+    ...entry,
+    responseHash: entry.responseHash ?? null,
+    tokensIn: entry.tokensIn ?? null,
+    tokensOut: entry.tokensOut ?? null,
+    temperature: entry.temperature ?? null,
+    seed: entry.seed ?? null,
+    cost: entry.cost ?? null,
+    calls: entry.calls ?? [],
+  }));
+}
+
+export function writeManifest(dir, manifest, outputs = []) {
+  mkdirSync(dir, { recursive: true });
+  const manifestPath = resolve(dir, 'manifest.json');
+  writeFileSync(manifestPath, JSON.stringify(manifest, null, 2) + '\n');
+  for (const { name, content } of outputs) {
+    writeFileSync(resolve(dir, name), content);
+  }
+  return manifestPath;
+}