papergod 0.1.0

package/src/server/text-extraction.js

@@ -0,0 +1,149 @@
+import { randomUUID } from 'crypto';
+import { createAgentRun, updateAgentRun } from './project-resources.js';
+import { runWritingAgent } from './agent-adapters.js';
+
+const MAX_TEXT_CHARS = 2_000_000;
+const MAX_CANDIDATES = 16;
+const SENTINEL = '[[PAPERGOD_PATTERN_EXTRACTION]]';
+
+const ACADEMIC_PATTERN = /\b(we (propose|show|demonstrate|present|introduce|develop|provide|examine|investigate|consider|find|report|observe|argue|hypothesize|conclude|explore|evaluate|compare|analyse|analyze|design|implement|establish|highlight|emphasize|suggest)|it (is|has been|was) (shown|found|demonstrated|observed|reported)|can be used to|plays (a|an) (key|important|crucial|central|significant) role|has been (widely|extensively|successfully) (used|studied|applied|adopted)|is one of the most|the results (indicate|suggest|show|demonstrate)|this (suggests|indicates|implies) that|our (results|findings|experiments|analysis|approach|method|framework|model)|in this (paper|work|study|article|section)|a wide range of|of particular interest|we (thus|therefore|consequently)|as a result,|in contrast,|compared with|consistent with|in addition,|furthermore,|moreover,|nevertheless,|on the other hand)\b/i;
+
+function now() { return new Date().toISOString(); }
+function id(prefix) { return `${prefix}_${randomUUID()}`; }
+function clean(value) { return typeof value === 'string' ? value.trim() : ''; }
+function problem(message, status = 400, code = '') {
+  const error = new Error(message);
+  error.status = status;
+  if (code) error.code = code;
+  return error;
+}
+
+function cleanPlainText(text) {
+  return String(text || '')
+    .replace(/[\u0000-\u0008\u000b\u000c\u000e-\u001f]/g, ' ')
+    .replace(/\s+/g, ' ')
+    .trim();
+}
+
+function splitSentences(text) {
+  return cleanPlainText(text)
+    .split(/(?<=[.!?])\s+/)
+    .map((sentence) => sentence.trim())
+    .filter((sentence) => sentence.length >= 30 && sentence.length <= 450);
+}
+
+function generalizeTemplate(sentence) {
+  let template = sentence;
+  let slotIndex = 0;
+  const slotify = (hint) => { slotIndex += 1; return `{slot${slotIndex}}`; };
+  template = template.replace(/\(([^)]*)\)/g, (_match, inner) => slotify(clean(inner).slice(0, 60)));
+  template = template.replace(/\b[A-Z][a-z]+(?:\s+[A-Z][a-z]+){1,3}\b/g, (word) => slotify(word));
+  template = template.replace(/\b\d+(?:[.,]\d+)+\b/g, () => slotify('number'));
+  template = template.replace(/\b\d+\b/g, () => slotify('number'));
+  template = template.replace(/['"“”‘’]/g, '');
+  return clean(template);
+}
+
+function detectSlots(template) {
+  return [...template.matchAll(/\{([a-zA-Z][\w-]*)\}/g)]
+    .map((match) => match[1])
+    .filter((name, index, names) => names.indexOf(name) === index)
+    .map((name) => ({ name, description: 'Replace with the concrete expression from your own writing', required: true }));
+}
+
+export function composeMockTextCandidates(text, source) {
+  const sentences = splitSentences(text);
+  const patterns = [];
+  for (const sentence of sentences) {
+    if (!ACADEMIC_PATTERN.test(sentence)) continue;
+    const template = generalizeTemplate(sentence);
+    if (!template) continue;
+    const slots = detectSlots(template);
+    patterns.push({
+      kind: 'sentence-patterns',
+      value: {
+        name: `PDF pattern ${patterns.length + 1}`,
+        template,
+        description: 'Reusable academic pattern extracted from a PDF by deterministic rules (academic framing phrase + slot generalization).',
+        tags: ['extracted', 'pdf'],
+        sectionTypes: [],
+        slots,
+        source,
+      },
+    });
+    if (patterns.length >= MAX_CANDIDATES) break;
+  }
+  return { patterns, vocabulary: [] };
+}
+
+function buildExternalPrompt(text, prompt, source) {
+  return `You are extracting reusable academic sentence patterns from a research PDF. Use only the supplied text. Return a list of suggestions where each suggestion has: originalText = one exact sentence from the text, suggestedText = a generalized template with concrete entities replaced by {slot1}, {slot2}, ... and description = when a writer should use this pattern. Prefer sentences with academic framing phrases (we propose, it has been shown, plays a key role, the results indicate, etc.). Extract ${MAX_CANDIDATES} patterns or fewer. Return exactly one suggestion that replaces the entire text ${SENTINEL} with the JSON array of {originalText, suggestedText, description} entries.
+
+Additional instruction: ${prompt || 'Focus on framing and transition patterns that are widely reusable.'}
+
+Source: ${source}
+
+Document text:
+<document>
+${text.slice(0, 400_000)}
+</document>`;
+}
+
+export async function extractTextCandidates(workspaceRoot, input = {}, options = {}) {
+  const text = typeof input.text === 'string' ? input.text : '';
+  if (!text.trim()) throw problem('PDF text is required');
+  if (text.length > MAX_TEXT_CHARS) throw problem(`PDF text exceeds ${MAX_TEXT_CHARS} characters`);
+  const source = clean(input.source) || 'PDF document';
+  const provider = options.provider || 'mock';
+  if (provider === 'mock') {
+    const candidates = composeMockTextCandidates(text, source);
+    return { candidates, provider, note: 'Mock extraction uses deterministic rules; for higher-quality patterns run with a configured external Agent.' };
+  }
+  const startedAt = now();
+  const run = await createAgentRun(workspaceRoot, {
+    provider, operation: 'extract-patterns', status: 'running', prompt: input.prompt || 'Extract reusable academic sentence patterns.',
+    input: JSON.stringify({ source, characters: text.length }), output: '', error: '', startedAt, finishedAt: '',
+  });
+  try {
+    const result = await runWritingAgent(provider, {
+      content: SENTINEL,
+      prompt: buildExternalPrompt(text, input.prompt, source),
+      resourceContext: '', resourceIds: [],
+    }, { workspaceRoot, commands: options.commands || {}, signal: options.signal });
+    const proposal = result.suggestions?.find((item) => item.originalText === SENTINEL) || result.suggestions?.[0];
+    if (!proposal?.suggestedText?.trim()) throw problem('Agent did not return extracted patterns', 502);
+    let entries = [];
+    try {
+      const parsed = JSON.parse(proposal.suggestedText);
+      entries = Array.isArray(parsed) ? parsed : [];
+    } catch {
+      const fence = proposal.suggestedText.match(/```(?:json)?\s*([\s\S]*?)```/i);
+      try { entries = JSON.parse(fence?.[1] || proposal.suggestedText); } catch { entries = []; }
+    }
+    const patterns = entries.slice(0, MAX_CANDIDATES).map((entry, index) => {
+      const template = clean(entry.suggestedText || entry.template || '');
+      const original = clean(entry.originalText || '');
+      if (!template) return null;
+      const slots = detectSlots(template);
+      return {
+        kind: 'sentence-patterns',
+        value: {
+          name: `PDF pattern ${index + 1}`,
+          template,
+          description: clean(entry.description) || `Extracted from ${source}${original ? ` — original: ${original.slice(0, 160)}` : ''}`,
+          tags: ['extracted', 'pdf'],
+          sectionTypes: [],
+          slots,
+          source,
+        },
+      };
+    }).filter(Boolean);
+    if (!patterns.length) throw problem('Agent returned no usable patterns', 502);
+    const finishedAt = now();
+    await updateAgentRun(workspaceRoot, run.id, { status: 'complete', output: JSON.stringify({ count: patterns.length, summary: patterns[0].value.template }), finishedAt });
+    return { candidates: { patterns, vocabulary: [] }, provider, note: '', runId: run.id };
+  } catch (error) {
+    await updateAgentRun(workspaceRoot, run.id, { status: 'failed', error: String(error?.message || 'Extraction failed').slice(0, 4000), finishedAt: now() });
+    throw error;
+  }
+}

package/src/server/workspace-browser.js

@@ -0,0 +1,49 @@
+import { homedir } from 'os';
+import { isAbsolute, relative, resolve, sep } from 'path';
+import { readdir, realpath, stat } from 'fs/promises';
+
+function isWithin(root, candidate) {
+  if (root === candidate) return true;
+  const rel = relative(root, candidate);
+  return rel !== '..' && !rel.startsWith(`..${sep}`) && !isAbsolute(rel);
+}
+
+async function canonical(path) {
+  try { return await realpath(path); } catch { return resolve(path); }
+}
+
+async function isGitRepository(path) {
+  try { return (await stat(resolve(path, '.git'))).isDirectory() || (await stat(resolve(path, '.git'))).isFile(); }
+  catch { return false; }
+}
+
+export async function browseWorkspaceDirectories(requestedPath = '', { root = homedir() } = {}) {
+  const browseRoot = await canonical(resolve(root));
+  const requested = typeof requestedPath === 'string' ? requestedPath.trim() : '';
+  const lexical = requested ? resolve(browseRoot, requested) : browseRoot;
+  const candidate = await canonical(lexical);
+  if (!isWithin(browseRoot, candidate)) {
+    throw Object.assign(new Error('Folder browsing is restricted to your home directory. Paste an absolute path to use another location.'), { status: 403, code: 'BROWSE_OUTSIDE_ROOT' });
+  }
+  let info;
+  try { info = await stat(candidate); }
+  catch (error) {
+    throw Object.assign(new Error(error.code === 'EACCES' ? 'Permission denied.' : 'Folder is unavailable.'), { status: 400, code: error.code || 'BROWSE_FAILED' });
+  }
+  if (!info.isDirectory()) throw Object.assign(new Error('The selected path is not a folder.'), { status: 400, code: 'INVALID_WORKSPACE_PATH' });
+  let entries;
+  try { entries = await readdir(candidate, { withFileTypes: true }); }
+  catch (error) {
+    throw Object.assign(new Error(error.code === 'EACCES' ? 'Permission denied.' : error.message), { status: 400, code: error.code || 'BROWSE_FAILED' });
+  }
+  const directories = entries.filter((entry) => entry.isDirectory() && !entry.name.startsWith('.')).sort((a, b) => a.name.localeCompare(b.name));
+  return {
+    rootPath: browseRoot,
+    currentPath: candidate,
+    parentPath: candidate === browseRoot ? null : resolve(candidate, '..'),
+    entries: await Promise.all(directories.map(async (entry) => {
+      const path = resolve(candidate, entry.name);
+      return { name: entry.name, path, git: await isGitRepository(path) };
+    })),
+  };
+}

package/src/server/workspace-registry.js

@@ -0,0 +1,143 @@
+import { createHash } from 'crypto';
+import { homedir } from 'os';
+import { basename, dirname, isAbsolute, join, resolve } from 'path';
+import { mkdir, readFile, realpath, rename, stat, writeFile } from 'fs/promises';
+
+export const DEFAULT_WORKSPACE_REGISTRY_FILE = join(homedir(), '.papergod', 'workspaces.json');
+
+function workspaceId(path) {
+  return `workspace_${createHash('sha256').update(path).digest('hex').slice(0, 16)}`;
+}
+
+async function canonicalDirectory(path) {
+  if (typeof path !== 'string' || !path.trim() || path.includes('\0')) {
+    throw Object.assign(new Error('Workspace path is required.'), { status: 400, code: 'INVALID_WORKSPACE_PATH' });
+  }
+  if (!isAbsolute(path.trim())) {
+    throw Object.assign(new Error('Use an absolute folder path.'), { status: 400, code: 'INVALID_WORKSPACE_PATH' });
+  }
+  const target = resolve(path.trim());
+  let info;
+  try {
+    info = await stat(target);
+  } catch (error) {
+    if (error.code === 'ENOENT') {
+      throw Object.assign(new Error('The selected folder does not exist.'), { status: 404, code: 'WORKSPACE_NOT_FOUND' });
+    }
+    throw error;
+  }
+  if (!info.isDirectory()) {
+    throw Object.assign(new Error('The selected path is not a folder.'), { status: 400, code: 'INVALID_WORKSPACE_PATH' });
+  }
+  return await realpath(target);
+}
+
+function emptyRegistry() {
+  return { version: 1, activePath: '', workspaces: [] };
+}
+
+async function readRegistry(file) {
+  try {
+    const parsed = JSON.parse(await readFile(file, 'utf8'));
+    return {
+      version: 1,
+      activePath: typeof parsed.activePath === 'string' ? parsed.activePath : '',
+      workspaces: Array.isArray(parsed.workspaces) ? parsed.workspaces.filter((item) => item && typeof item.path === 'string') : [],
+    };
+  } catch (error) {
+    if (error.code === 'ENOENT' || error instanceof SyntaxError) return emptyRegistry();
+    throw error;
+  }
+}
+
+async function writeRegistry(file, registry) {
+  await mkdir(dirname(file), { recursive: true });
+  const temporary = `${file}.${process.pid}.tmp`;
+  await writeFile(temporary, `${JSON.stringify(registry, null, 2)}\n`, 'utf8');
+  await rename(temporary, file);
+}
+
+export function createWorkspaceRegistry({ file = DEFAULT_WORKSPACE_REGISTRY_FILE } = {}) {
+  let operationQueue = Promise.resolve();
+  const serialize = (operation) => {
+    const result = operationQueue.then(operation, operation);
+    operationQueue = result.catch(() => {});
+    return result;
+  };
+
+  async function add(path, { activate = false } = {}) {
+    const canonical = await canonicalDirectory(path);
+    const registry = await readRegistry(file);
+    const timestamp = new Date().toISOString();
+    let entry = registry.workspaces.find((item) => item.path === canonical);
+    if (!entry) {
+      entry = { id: workspaceId(canonical), name: basename(canonical) || canonical, path: canonical, addedAt: timestamp, lastOpenedAt: timestamp };
+      registry.workspaces.push(entry);
+    }
+    if (activate) {
+      entry.lastOpenedAt = timestamp;
+      registry.activePath = canonical;
+    }
+    await writeRegistry(file, registry);
+    return { ...entry };
+  }
+
+  async function activate(idOrPath) {
+    const registry = await readRegistry(file);
+    let entry = registry.workspaces.find((item) => item.id === idOrPath);
+    if (!entry && typeof idOrPath === 'string' && isAbsolute(idOrPath)) {
+      const canonical = await canonicalDirectory(idOrPath);
+      entry = registry.workspaces.find((item) => item.path === canonical);
+    }
+    if (!entry) throw Object.assign(new Error('Workspace is not registered.'), { status: 404, code: 'WORKSPACE_NOT_FOUND' });
+    entry.path = await canonicalDirectory(entry.path);
+    entry.name = basename(entry.path) || entry.path;
+    entry.lastOpenedAt = new Date().toISOString();
+    registry.activePath = entry.path;
+    await writeRegistry(file, registry);
+    return { ...entry };
+  }
+
+  async function getActive() {
+    const registry = await readRegistry(file);
+    if (!registry.activePath) return null;
+    try {
+      const canonical = await canonicalDirectory(registry.activePath);
+      const entry = registry.workspaces.find((item) => item.path === registry.activePath || item.path === canonical);
+      return entry
+        ? { ...entry, path: canonical, name: basename(canonical) || canonical }
+        : { id: workspaceId(canonical), name: basename(canonical) || canonical, path: canonical };
+    } catch {
+      return null;
+    }
+  }
+
+  async function list(activePath = '') {
+    const registry = await readRegistry(file);
+    const effectiveActive = activePath || registry.activePath;
+    const workspaces = [];
+    let changed = false;
+    for (const entry of registry.workspaces) {
+      try {
+        const canonical = await canonicalDirectory(entry.path);
+        if (canonical !== entry.path) changed = true;
+        workspaces.push({ ...entry, path: canonical, name: basename(canonical) || canonical, available: true, active: canonical === effectiveActive });
+      } catch {
+        workspaces.push({ ...entry, available: false, active: entry.path === effectiveActive });
+      }
+    }
+    if (changed) {
+      registry.workspaces = workspaces.map(({ available: _available, active: _active, ...entry }) => entry);
+      await writeRegistry(file, registry);
+    }
+    return workspaces.sort((a, b) => Number(b.active) - Number(a.active) || String(b.lastOpenedAt).localeCompare(String(a.lastOpenedAt)));
+  }
+
+  return {
+    add: (...args) => serialize(() => add(...args)),
+    activate: (...args) => serialize(() => activate(...args)),
+    getActive: (...args) => serialize(() => getActive(...args)),
+    list: (...args) => serialize(() => list(...args)),
+    file,
+  };
+}

package/src/server/workspace-terminal.js

@@ -0,0 +1,99 @@
+import { randomUUID } from 'crypto';
+import { basename } from 'path';
+import { spawn as spawnPty } from 'node-pty';
+
+const MAX_HISTORY = 200_000;
+const MAX_INPUT = 64_000;
+
+function shellLaunch(env = process.env, platform = process.platform) {
+  if (platform === 'win32') return { command: env.ComSpec || 'cmd.exe', args: [] };
+  const command = env.SHELL || '/bin/bash';
+  return { command, args: ['bash', 'zsh', 'fish', 'ksh'].includes(basename(command)) ? ['-l'] : [] };
+}
+
+export function createWorkspaceTerminalManager({ env = process.env, platform = process.platform } = {}) {
+  const sessions = new Map();
+  const workspaceSessions = new Map();
+
+  function publicSession(session) {
+    return { id: session.id, workspace: session.workspace, status: session.status, pid: session.pty.pid, startedAt: session.startedAt, exitCode: session.exitCode };
+  }
+
+  function broadcast(session, event, payload) {
+    const body = `event: ${event}\ndata: ${JSON.stringify(payload)}\n\n`;
+    for (const response of session.listeners) response.write(body);
+  }
+
+  function start(workspace) {
+    const existingId = workspaceSessions.get(workspace);
+    const existing = existingId && sessions.get(existingId);
+    if (existing?.status === 'running') return publicSession(existing);
+    if (existing) sessions.delete(existing.id);
+    const launch = shellLaunch(env, platform);
+    const pty = spawnPty(launch.command, launch.args, {
+      name: 'xterm-256color', cols: 100, rows: 30, cwd: workspace,
+      env: { ...env, TERM: 'xterm-256color', COLORTERM: 'truecolor', TERM_PROGRAM: 'papergod' },
+    });
+    const session = { id: `terminal_${randomUUID()}`, workspace, pty, status: 'running', exitCode: null, startedAt: new Date().toISOString(), history: '', listeners: new Set() };
+    sessions.set(session.id, session);
+    workspaceSessions.set(workspace, session.id);
+    pty.onData((data) => {
+      session.history = `${session.history}${data}`.slice(-MAX_HISTORY);
+      broadcast(session, 'output', { data });
+    });
+    pty.onExit(({ exitCode, signal }) => {
+      session.status = 'exited';
+      session.exitCode = exitCode;
+      broadcast(session, 'exit', { exitCode, signal });
+      for (const response of session.listeners) response.end();
+      session.listeners.clear();
+    });
+    return publicSession(session);
+  }
+
+  function requireSession(id) {
+    const session = sessions.get(id);
+    if (!session) throw Object.assign(new Error('Terminal session not found.'), { status: 404, code: 'TERMINAL_NOT_FOUND' });
+    return session;
+  }
+
+  function attach(id, response) {
+    const session = requireSession(id);
+    session.listeners.add(response);
+    response.write(`event: ready\ndata: ${JSON.stringify({ session: publicSession(session), history: session.history })}\n\n`);
+    if (session.status !== 'running') response.write(`event: exit\ndata: ${JSON.stringify({ exitCode: session.exitCode })}\n\n`);
+    return () => session.listeners.delete(response);
+  }
+
+  function input(id, data) {
+    const session = requireSession(id);
+    if (session.status !== 'running') throw Object.assign(new Error('Terminal has exited.'), { status: 409, code: 'TERMINAL_EXITED' });
+    if (typeof data !== 'string' || data.length > MAX_INPUT) throw Object.assign(new Error('Terminal input must be a string up to 64 KB.'), { status: 400, code: 'INVALID_TERMINAL_INPUT' });
+    session.pty.write(data);
+  }
+
+  function resize(id, cols, rows) {
+    const session = requireSession(id);
+    if (!Number.isInteger(cols) || !Number.isInteger(rows) || cols < 10 || cols > 400 || rows < 4 || rows > 200) {
+      throw Object.assign(new Error('Invalid terminal dimensions.'), { status: 400, code: 'INVALID_TERMINAL_SIZE' });
+    }
+    if (session.status === 'running') session.pty.resize(cols, rows);
+  }
+
+  function close(id) {
+    const session = requireSession(id);
+    if (session.status === 'running') session.pty.kill();
+    sessions.delete(id);
+    if (workspaceSessions.get(session.workspace) === id) workspaceSessions.delete(session.workspace);
+    for (const response of session.listeners) response.end();
+    session.listeners.clear();
+  }
+
+  function closeAll() {
+    for (const id of [...sessions.keys()]) {
+      try { close(id); } catch {}
+    }
+  }
+
+  return { start, attach, input, resize, close, closeAll, get: (id) => publicSession(requireSession(id)) };
+}