papergod 0.1.0

package/src/server/paragraph-analysis.js

@@ -0,0 +1,300 @@
+import { loadProject } from './project-store.js';
+import { getNodeSourceContext, syncDocumentStructure } from './document-structure.js';
+import { findStructureNode, isAbbreviationAt, sentenceEndIndex } from './latex-structure.js';
+
+// ---------------------------------------------------------------------------
+// Descriptive statistics over sentence/paragraph lengths (word counts)
+// ---------------------------------------------------------------------------
+
+export const ANALYSIS_FORMULAS = [
+  { id: 'mean', name: 'Mean length', formula: 'μ = (1/n) · Σᵢ₌₁ⁿ xᵢ', description: 'Average sentence (or paragraph) length in words.' },
+  { id: 'stddev', name: 'Sample standard deviation', formula: 's = √( Σᵢ₌₁ⁿ (xᵢ − μ)² / (n − 1) )', description: 'Spread of lengths around the mean. n − 1 (Bessel) is used so s is an unbiased estimate of the population standard deviation.' },
+  { id: 'variance', name: 'Variance', formula: 's² = Σᵢ₌₁ⁿ (xᵢ − μ)² / (n − 1)', description: 'Squared standard deviation; mean squared deviation from the mean.' },
+  { id: 'median', name: 'Median', formula: 'median = middle value of sorted lengths (average of the two middle values when n is even)', description: 'Robust centre; half the units are shorter, half longer.' },
+  { id: 'range', name: 'Range', formula: 'R = max(x) − min(x)', description: 'Full spread between the longest and shortest unit.' },
+  { id: 'iqr', name: 'Interquartile range', formula: 'IQR = Q₃ − Q₁', description: 'Spread of the middle 50% of the data; robust to outliers.' },
+  { id: 'cv', name: 'Coefficient of variation', formula: 'CV = s / μ', description: 'Relative variation. Low CV means lengths cluster tightly around the mean — the signature of mechanically uniform text.' },
+  { id: 'delta', name: 'Adjacent change', formula: 'Δ = (1/(n−1)) · Σᵢ₌₁ⁿ⁻¹ |xᵢ₊₁ − xᵢ|', description: 'Average absolute jump between consecutive units; captures rhythm and variation from one sentence to the next.' },
+  { id: 'relativeDelta', name: 'Relative adjacent change', formula: 'Δ / μ', description: 'Adjacent change normalised by the mean, comparable across texts of different scales.' },
+  { id: 'variationIndex', name: 'Variation index', formula: 'VI = 100 · ( 0.6 · min(1, CV/0.5) + 0.4 · min(1, (Δ/μ)/0.8) )', description: '0–100 composite of CV and relative adjacent change. Higher values mean more rhythmic variation (organic prose); low values mean uniform, template-like writing.' },
+];
+
+function cleanText(value) {
+  return String(value || '').replace(/\s+/g, ' ').trim();
+}
+
+// Strip LaTeX markup before counting words and splitting sentences so that
+// commands and math do not inflate sentence lengths or break sentence pauses.
+function cleanLatexForText(value) {
+  return cleanText(value)
+    .replace(/(?<!\\)%.*$/gm, ' ')
+    .replace(/\\(?:cite|ref|label|footnote|emph|textbf|textit|texttt)\*?(?:\[[^\]]*\])?\{([^{}]*)\}/g, '$1')
+    .replace(/\\[a-zA-Z@]+\*?(?:\[[^\]]*\])?/g, ' ')
+    .replace(/[{}]/g, ' ')
+    .replace(/\$+[^$]*\$+/g, ' equation ');
+}
+
+export function wordCount(text) {
+  const cleaned = cleanLatexForText(text);
+  return cleaned ? cleaned.split(/\s+/).filter(Boolean).length : 0;
+}
+
+export function splitSentences(text) {
+  const source = cleanLatexForText(text);
+  if (!source) return [];
+  const sentences = [];
+  let sentenceStart = 0;
+  for (let cursor = 0; cursor < source.length; cursor += 1) {
+    const character = source[cursor];
+    if (!'.?!'.includes(character)) continue;
+    const previous = source[cursor - 1];
+    const boundary = sentenceEndIndex(source, cursor);
+    if (boundary === -1) continue; // glued to following word/command
+    let look = boundary;
+    while (look < source.length && /\s/.test(source[look])) look += 1;
+    const nextNonSpace = source[look];
+    if (/\d/.test(previous || '') && /\d/.test(nextNonSpace || '')) continue; // decimal
+    if (character === '.' && isAbbreviationAt(source, cursor)) continue; // e.g. i.e. cf.
+    if (look < source.length && /[a-z]/.test(nextNonSpace || '')) continue; // embedded quote continues
+    const sentence = cleanText(source.slice(sentenceStart, boundary));
+    if (sentence) sentences.push(sentence);
+    sentenceStart = boundary;
+    cursor = boundary - 1;
+  }
+  const tail = cleanText(source.slice(sentenceStart));
+  if (tail) sentences.push(tail);
+  return sentences;
+}
+
+function round(value, digits = 2) {
+  if (!Number.isFinite(value)) return 0;
+  const factor = 10 ** digits;
+  return Math.round(value * factor) / factor;
+}
+
+function mean(values) {
+  return values.reduce((sum, value) => sum + value, 0) / values.length;
+}
+
+function sampleStddev(values) {
+  if (values.length < 2) return 0;
+  const mu = mean(values);
+  return Math.sqrt(values.reduce((sum, value) => sum + (value - mu) ** 2, 0) / (values.length - 1));
+}
+
+function median(sorted) {
+  const n = sorted.length;
+  if (!n) return 0;
+  const mid = Math.floor(n / 2);
+  return n % 2 ? sorted[mid] : (sorted[mid - 1] + sorted[mid]) / 2;
+}
+
+function adjacentDelta(values) {
+  if (values.length < 2) return 0;
+  let sum = 0;
+  for (let index = 1; index < values.length; index += 1) sum += Math.abs(values[index] - values[index - 1]);
+  return sum / (values.length - 1);
+}
+
+export function analyzeLengths(values) {
+  const n = values.length;
+  if (!n) return { count: 0 };
+  const sorted = [...values].sort((a, b) => a - b);
+  const mu = mean(values);
+  const s = sampleStddev(values);
+  const delta = adjacentDelta(values);
+  const lowerHalf = sorted.slice(0, Math.floor(sorted.length / 2));
+  const upperHalf = sorted.slice(Math.ceil(sorted.length / 2));
+  return {
+    count: n,
+    mean: round(mu),
+    stddev: round(s),
+    variance: round(s * s),
+    min: sorted[0],
+    max: sorted[n - 1],
+    range: round(sorted[n - 1] - sorted[0]),
+    median: round(median(sorted)),
+    iqr: round(median(upperHalf) - median(lowerHalf)),
+    cv: round(mu ? s / mu : 0),
+    delta: round(delta),
+    relativeDelta: round(mu ? delta / mu : 0),
+  };
+}
+
+// ---------------------------------------------------------------------------
+// Variation index (0-100) and the mechanical-writing verdict
+// ---------------------------------------------------------------------------
+
+export function variationIndex(sentenceStats) {
+  const cvPart = Math.min(1, (sentenceStats.cv || 0) / 0.5);
+  const deltaPart = Math.min(1, (sentenceStats.relativeDelta || 0) / 0.8);
+  return {
+    score: Math.round((0.6 * cvPart + 0.4 * deltaPart) * 100),
+    cvPart: round(cvPart, 3),
+    deltaPart: round(deltaPart, 3),
+  };
+}
+
+export function mechanicalVerdict(score) {
+  // score is the variation index (0-100): LOW = uniform/template-like, HIGH = organic.
+  if (score >= 70) {
+    return {
+      label: 'highly-varied', title: 'Highly varied rhythm',
+      note: 'Sentence lengths change sharply and irregularly throughout. Strong rhythmic variation is typical of organic, edited prose.',
+    };
+  }
+  if (score >= 45) {
+    return {
+      label: 'varied', title: 'Varied rhythm',
+      note: 'Sentence lengths fluctuate noticeably, with meaningful jumps between adjacent sentences. This resembles natural human drafting.',
+    };
+  }
+  if (score >= 25) {
+    return {
+      label: 'uniform', title: 'Uniform rhythm',
+      note: 'Sentence lengths cluster tightly around the mean. The text reads steadily; consider whether that evenness matches an intentional style or a mechanical template.',
+    };
+  }
+  return {
+    label: 'highly-uniform', title: 'Highly uniform rhythm',
+    note: 'Sentence lengths barely vary and rarely jump between adjacent sentences. This is the pattern typical of template-generated text — but uniformity alone is not proof of AI writing; short, technical passages are naturally even.',
+  };
+}
+
+// ---------------------------------------------------------------------------
+// Analysis units
+// ---------------------------------------------------------------------------
+
+function analyzeTextUnit(text, { id = '', label = '' } = {}) {
+  const sentences = splitSentences(text);
+  const sentenceLengths = sentences.map((sentence, index) => ({
+    index, text: sentence, wordCount: wordCount(sentence), charCount: sentence.length,
+  }));
+  const stats = analyzeLengths(sentenceLengths.map((sentence) => sentence.wordCount));
+  const variation = variationIndex(stats);
+  return {
+    unit: { id, label, text: cleanText(text), sentenceCount: sentences.length, wordCount: wordCount(text) },
+    sentences: sentenceLengths,
+    stats,
+    variation,
+    verdict: mechanicalVerdict(variation.score),
+    formulas: ANALYSIS_FORMULAS,
+  };
+}
+
+function flattenParagraphs(sections) {
+  const result = [];
+  const visit = (nodes, section) => {
+    for (const node of nodes || []) {
+      if (node.type === 'paragraph') result.push({ node, section });
+      visit(node.children, section);
+    }
+  };
+  for (const section of sections || []) visit([section], section);
+  return result;
+}
+
+function summarizeUnit(analysis) {
+  return {
+    id: analysis.unit.id,
+    label: analysis.unit.label,
+    text: analysis.unit.text,
+    sentenceCount: analysis.unit.sentenceCount,
+    wordCount: analysis.unit.wordCount,
+    stats: analysis.stats,
+    variation: analysis.variation,
+    verdict: analysis.verdict,
+  };
+}
+
+function analyzeDocumentStructure(document) {
+  const paragraphs = flattenParagraphs(document.sections);
+  const paragraphAnalyses = paragraphs.map(({ node, section }) => ({
+    ...summarizeUnit(analyzeTextUnit(node.text, { id: node.id, label: node.summary || `Paragraph in ${section?.title || 'document'}` })),
+    section: section?.title || '',
+  }));
+  const allSentenceLengths = paragraphAnalyses.flatMap((paragraph) => {
+    const sentences = splitSentences(paragraph.text);
+    return sentences.map((sentence) => wordCount(sentence));
+  });
+  const paragraphLengths = paragraphAnalyses.map((paragraph) => paragraph.wordCount);
+  const paragraphSentenceCounts = paragraphAnalyses.map((paragraph) => paragraph.sentenceCount);
+  const sentenceStats = analyzeLengths(allSentenceLengths);
+  const paragraphStats = analyzeLengths(paragraphLengths);
+  const variation = variationIndex(sentenceStats);
+  return {
+    unit: {
+      id: document.id,
+      label: document.title || document.file,
+      text: '',
+      paragraphCount: paragraphAnalyses.length,
+      sentenceCount: allSentenceLengths.length,
+      wordCount: paragraphLengths.reduce((sum, value) => sum + value, 0),
+    },
+    paragraphs: paragraphAnalyses,
+    sentenceStats,
+    paragraphStats,
+    paragraphSentenceStats: analyzeLengths(paragraphSentenceCounts),
+    variation,
+    verdict: mechanicalVerdict(variation.score),
+    formulas: ANALYSIS_FORMULAS,
+  };
+}
+
+// ---------------------------------------------------------------------------
+// Public API entry
+// ---------------------------------------------------------------------------
+
+function error(message, status = 400) {
+  const value = new Error(message);
+  value.status = status;
+  return value;
+}
+
+export async function analyzeStructure(workspaceRoot, { documentId, nodeId, content } = {}) {
+  if (typeof content === 'string' && content.trim()) {
+    return { kind: 'selection', ...analyzeTextUnit(content, { label: 'Selection' }) };
+  }
+  let project = await loadProject(workspaceRoot);
+  let document = project.documents.find((item) => item.id === documentId);
+  if (!document) {
+    document = project.documents[0];
+    if (!document) throw error('No document found in this workspace', 404);
+  }
+  await syncDocumentStructure(workspaceRoot, document.file);
+  project = await loadProject(workspaceRoot);
+  document = project.documents.find((item) => item.id === document.id);
+
+  if (typeof nodeId === 'string' && nodeId) {
+    const context = await getNodeSourceContext(workspaceRoot, nodeId);
+    if (context.node.type === 'sentence') {
+      const paragraph = context.node.parentId ? findStructureNode(context.document, context.node.parentId) : null;
+      if (paragraph?.type === 'paragraph') {
+        return { kind: 'paragraph', section: context.section?.title || '', ...analyzeTextUnit(paragraph.text, { id: paragraph.id, label: paragraph.summary || 'Paragraph' }) };
+      }
+      return { kind: 'sentence', section: context.section?.title || '', ...analyzeTextUnit(context.node.text, { id: context.node.id, label: 'Sentence' }) };
+    }
+    if (context.node.type === 'paragraph') {
+      return { kind: 'paragraph', section: context.section?.title || '', ...analyzeTextUnit(context.node.text, { id: context.node.id, label: context.node.summary || 'Paragraph' }) };
+    }
+    if (context.node.type === 'section') {
+      const paragraphs = flattenParagraphs([context.node]);
+      const paragraphAnalyses = paragraphs.map(({ node }) => summarizeUnit(analyzeTextUnit(node.text, { id: node.id, label: node.summary || `Paragraph in ${context.node.title}` })));
+      const sentenceStats = analyzeLengths(paragraphAnalyses.flatMap((paragraph) => splitSentences(paragraph.text).map((sentence) => wordCount(sentence))));
+      const paragraphStats = analyzeLengths(paragraphAnalyses.map((paragraph) => paragraph.wordCount));
+      const variation = variationIndex(sentenceStats);
+      return {
+        kind: 'section',
+        section: context.node.title,
+        unit: { id: context.node.id, label: context.node.title, paragraphCount: paragraphAnalyses.length, sentenceCount: sentenceStats.count, wordCount: paragraphStats.count ? paragraphStats.mean * paragraphStats.count : 0 },
+        paragraphs: paragraphAnalyses,
+        sentenceStats,
+        paragraphStats,
+        variation,
+        verdict: mechanicalVerdict(variation.score),
+        formulas: ANALYSIS_FORMULAS,
+      };
+    }
+  }
+  return { kind: 'document', ...analyzeDocumentStructure(document) };
+}

package/src/server/project-resources.js

@@ -0,0 +1,290 @@
+import { randomUUID } from 'crypto';
+import { loadProject, updateProject } from './project-store.js';
+
+function timestamp() {
+  return new Date().toISOString();
+}
+
+function id(prefix) {
+  return `${prefix}_${randomUUID()}`;
+}
+
+function text(value, fallback = '') {
+  return typeof value === 'string' ? value : fallback;
+}
+
+function strings(value) {
+  return Array.isArray(value) ? value.filter((item) => typeof item === 'string' && item.trim()) : [];
+}
+
+function resourceError(message, status = 400) {
+  const error = new Error(message);
+  error.status = status;
+  return error;
+}
+
+const libraryKinds = {
+  corpora: {
+    prefix: 'corpus',
+    collection: (project) => project.libraries.corpora,
+    build(input, existing) {
+      const time = timestamp();
+      return {
+        id: existing?.id || id('corpus'),
+        name: text(input.name),
+        description: text(input.description),
+        content: text(input.content),
+        source: text(input.source),
+        tags: strings(input.tags),
+        createdAt: existing?.createdAt || time,
+        updatedAt: time,
+      };
+    },
+  },
+  'sentence-patterns': {
+    prefix: 'pattern',
+    collection: (project) => project.libraries.sentencePatterns,
+    build(input, existing) {
+      const time = timestamp();
+      return {
+        id: existing?.id || id('pattern'),
+        name: text(input.name),
+        template: text(input.template),
+        description: text(input.description),
+        source: text(input.source),
+        tags: strings(input.tags),
+        sectionTypes: strings(input.sectionTypes),
+        slots: Array.isArray(input.slots) ? input.slots.map((slot) => ({
+          name: text(slot?.name),
+          description: text(slot?.description),
+          required: slot?.required !== false,
+        })) : [],
+        createdAt: existing?.createdAt || time,
+        updatedAt: time,
+      };
+    },
+  },
+};
+
+function vocabularyConfig(scope) {
+  if (!['global', 'session'].includes(scope)) throw resourceError('Vocabulary scope must be global or session');
+  return {
+    collection: (project) => project.libraries.vocabulary[scope],
+    build(input, existing) {
+      const time = timestamp();
+      return {
+        id: existing?.id || id('vocabulary'),
+        term: text(input.term),
+        preferred: text(input.preferred),
+        definition: text(input.definition),
+        source: text(input.source),
+        alternatives: strings(input.alternatives),
+        examples: strings(input.examples),
+        tags: strings(input.tags),
+        createdAt: existing?.createdAt || time,
+        updatedAt: time,
+      };
+    },
+  };
+}
+
+function libraryConfig(kind, scope) {
+  if (kind === 'vocabulary') return vocabularyConfig(scope);
+  const config = libraryKinds[kind];
+  if (!config) throw resourceError('Unknown library kind', 404);
+  return config;
+}
+
+export async function getLibraries(workspaceRoot) {
+  return (await loadProject(workspaceRoot)).libraries;
+}
+
+export async function createLibraryResource(workspaceRoot, kind, scope, input = {}) {
+  const config = libraryConfig(kind, scope);
+  const item = config.build(input);
+  await updateProject(workspaceRoot, (project) => config.collection(project).push(item));
+  return item;
+}
+
+export async function updateLibraryResource(workspaceRoot, kind, scope, resourceId, input = {}) {
+  const config = libraryConfig(kind, scope);
+  const { result } = await updateProject(workspaceRoot, (project) => {
+    const collection = config.collection(project);
+    const index = collection.findIndex((item) => item.id === resourceId);
+    if (index === -1) throw resourceError('Library resource not found', 404);
+    const updated = config.build({ ...collection[index], ...input }, collection[index]);
+    collection[index] = updated;
+    return updated;
+  });
+  return result;
+}
+
+export async function deleteLibraryResource(workspaceRoot, kind, scope, resourceId) {
+  const config = libraryConfig(kind, scope);
+  await updateProject(workspaceRoot, (project) => {
+    const collection = config.collection(project);
+    const index = collection.findIndex((item) => item.id === resourceId);
+    if (index === -1) throw resourceError('Library resource not found', 404);
+    collection.splice(index, 1);
+  });
+}
+
+function buildTarget(input = {}) {
+  return {
+    type: text(input.type, 'document'),
+    id: text(input.id),
+    start: Number.isInteger(input.start) ? input.start : 0,
+    end: Number.isInteger(input.end) ? input.end : 0,
+    quote: text(input.quote),
+  };
+}
+
+function buildAnnotation(input, existing) {
+  const time = timestamp();
+  return {
+    id: existing?.id || id('annotation'),
+    documentId: text(input.documentId),
+    target: buildTarget(input.target),
+    category: text(input.category, 'other'),
+    severity: text(input.severity, 'info'),
+    body: text(input.body),
+    suggestedFix: text(input.suggestedFix),
+    status: text(input.status, 'open'),
+    order: Number.isInteger(input.order) ? input.order : existing?.order,
+    dependsOn: strings(input.dependsOn),
+    source: {
+      type: text(input.source?.type, 'user'),
+      actor: text(input.source?.actor),
+    },
+    createdAt: existing?.createdAt || time,
+    updatedAt: time,
+  };
+}
+
+export async function listAnnotations(workspaceRoot, documentId) {
+  const annotations = (await loadProject(workspaceRoot)).annotations;
+  return documentId ? annotations.filter((item) => item.documentId === documentId) : annotations;
+}
+
+export async function createAnnotation(workspaceRoot, input = {}) {
+  const item = buildAnnotation(input);
+  await updateProject(workspaceRoot, (project) => project.annotations.push(item));
+  return item;
+}
+
+export async function updateAnnotation(workspaceRoot, annotationId, input = {}) {
+  const { result } = await updateProject(workspaceRoot, (project) => {
+    const index = project.annotations.findIndex((item) => item.id === annotationId);
+    if (index === -1) throw resourceError('Annotation not found', 404);
+    const existing = project.annotations[index];
+    const updated = buildAnnotation({
+      ...existing,
+      ...input,
+      target: { ...existing.target, ...input.target },
+      source: { ...existing.source, ...input.source },
+    }, existing);
+    project.annotations[index] = updated;
+    return updated;
+  });
+  return result;
+}
+
+export async function deleteAnnotation(workspaceRoot, annotationId) {
+  await updateProject(workspaceRoot, (project) => {
+    const index = project.annotations.findIndex((item) => item.id === annotationId);
+    if (index === -1) throw resourceError('Annotation not found', 404);
+    project.annotations.splice(index, 1);
+  });
+}
+
+function buildRevision(input, existing) {
+  const time = timestamp();
+  return {
+    id: existing?.id || id('revision'),
+    documentId: text(input.documentId),
+    title: text(input.title),
+    summary: text(input.summary),
+    status: text(input.status, 'draft'),
+    annotationIds: strings(input.annotationIds),
+    changes: Array.isArray(input.changes) ? input.changes.map((change) => ({
+      id: text(change?.id) || id('change'),
+      target: buildTarget(change?.target),
+      before: text(change?.before),
+      after: text(change?.after),
+      reason: text(change?.reason),
+      status: text(change?.status, 'proposed'),
+    })) : [],
+    createdAt: existing?.createdAt || time,
+    updatedAt: time,
+  };
+}
+
+export async function listRevisions(workspaceRoot, documentId) {
+  const revisions = (await loadProject(workspaceRoot)).revisions;
+  return documentId ? revisions.filter((item) => item.documentId === documentId) : revisions;
+}
+
+export async function createRevision(workspaceRoot, input = {}) {
+  const item = buildRevision(input);
+  await updateProject(workspaceRoot, (project) => project.revisions.push(item));
+  return item;
+}
+
+export async function updateRevision(workspaceRoot, revisionId, input = {}) {
+  const { result } = await updateProject(workspaceRoot, (project) => {
+    const index = project.revisions.findIndex((item) => item.id === revisionId);
+    if (index === -1) throw resourceError('Revision not found', 404);
+    const updated = buildRevision({ ...project.revisions[index], ...input }, project.revisions[index]);
+    project.revisions[index] = updated;
+    return updated;
+  });
+  return result;
+}
+
+export async function deleteRevision(workspaceRoot, revisionId) {
+  await updateProject(workspaceRoot, (project) => {
+    const index = project.revisions.findIndex((item) => item.id === revisionId);
+    if (index === -1) throw resourceError('Revision not found', 404);
+    project.revisions.splice(index, 1);
+  });
+}
+
+function buildAgentRun(input, existing) {
+  const time = timestamp();
+  return {
+    id: existing?.id || id('agent_run'),
+    provider: text(input.provider, 'mock'),
+    operation: text(input.operation),
+    status: text(input.status, 'queued'),
+    prompt: text(input.prompt),
+    input: text(input.input),
+    output: text(input.output),
+    error: text(input.error),
+    startedAt: text(input.startedAt),
+    finishedAt: text(input.finishedAt),
+    createdAt: existing?.createdAt || time,
+    updatedAt: time,
+  };
+}
+
+export async function createAgentRun(workspaceRoot, input = {}) {
+  const item = buildAgentRun(input);
+  await updateProject(workspaceRoot, (project) => project.agentRuns.push(item));
+  return item;
+}
+
+export async function updateAgentRun(workspaceRoot, runId, input = {}) {
+  const { result } = await updateProject(workspaceRoot, (project) => {
+    const index = project.agentRuns.findIndex((item) => item.id === runId);
+    if (index === -1) throw resourceError('Agent run not found', 404);
+    const existing = project.agentRuns[index];
+    const updated = buildAgentRun({ ...existing, ...input }, existing);
+    project.agentRuns[index] = updated;
+    return updated;
+  });
+  return result;
+}
+
+export async function listAgentRuns(workspaceRoot) {
+  return (await loadProject(workspaceRoot)).agentRuns;
+}