threadshelf 1.2.0

package/SECURITY.md

@@ -0,0 +1,75 @@
+# Security And Privacy
+
+ThreadShelf is designed for local private archives. It does not require a cloud vector database or external embedding API for the main search path.
+
+## Private Data
+
+Do not commit:
+
+- Real chat exports.
+- Uploaded source files.
+- LanceDB databases.
+- `.collections.json`.
+- `.threadshelf/` (generation settings and optional llama.cpp tools).
+- Logs.
+- Temp folders.
+- Screenshots containing private conversations.
+
+The repo's committed fixtures should be synthetic and anonymized.
+
+## Local Server Exposure
+
+By default the server binds only to `127.0.0.1`. Keep that default for private
+single-machine use:
+
+```bash
+$env:HOST='127.0.0.1'
+npm start
+```
+
+Setting `HOST=0.0.0.0` exposes the unauthenticated API to the local network. Do
+not do this on an untrusted network.
+
+The browser UI uses same-origin API requests and does not enable cross-origin
+access by default.
+
+## Experimental Generation Boundary
+
+Conversation generation is **Experimental Beta**. Managed `llama-server`
+processes bind to an ephemeral `127.0.0.1` port, configured llama.cpp URLs must
+be loopback-only, and selected GGUF files must be under configured model roots.
+
+OpenRouter is an explicit external exception to the local data path. Sending a
+continuation transmits the selected archive's user/assistant turns and prompt to
+OpenRouter and its routed provider; archived thinking is excluded. The UI
+marks the provider as **OpenRouter · external**, adds an `off-device` chip to the
+model button, and repeats the boundary in the composer. ZDR-only routing and
+provider data-collection denial are optional controls, but users must still treat
+every OpenRouter request as disclosure to an external service.
+
+Prefer `OPENROUTER_API_KEY`. Keys entered in Settings remain in process memory
+only and are never returned by the API or persisted to generation config.
+
+## MCP Exposure
+
+The MCP server gives connected MCP clients access to indexed local chat data. Only configure it in clients you trust.
+
+## Reporting Issues
+
+If you find a vulnerability or privacy leak:
+
+- Do not include real private exports in the report.
+- Describe the issue with synthetic examples.
+- Include affected routes/tools and reproduction steps.
+
+## Dependency Notes
+
+The app uses:
+
+- Express for the local HTTP server.
+- Multer for uploads.
+- LanceDB for local vector storage.
+- Xenova Transformers for local embeddings.
+- Playwright for browser tests.
+
+Review dependency updates before publishing public releases.

package/bin/threadshelf-mcp.js

@@ -0,0 +1,12 @@
+#!/usr/bin/env node
+/**
+ * `threadshelf-mcp` stdio entrypoint for MCP clients (Claude Desktop, etc.).
+ * Kept separate from the compiled module so the server's own "am I the
+ * entrypoint?" check is not needed here - we start it explicitly.
+ */
+import { runServer } from '../dist/mcp/server.js';
+import { startIndexRecovery } from '../dist/src/store.js';
+
+const stopRecovery = startIndexRecovery();
+process.stdin.once('end', stopRecovery);
+runServer();

package/bin/threadshelf.js

@@ -0,0 +1,87 @@
+#!/usr/bin/env node
+/**
+ * `npx threadshelf` entrypoint.
+ *
+ * Plain JavaScript on purpose: the published package must not need tsx or a
+ * TypeScript toolchain at runtime. It parses a couple of flags, then hands over
+ * to the compiled server in dist/.
+ */
+import { createRequire } from 'node:module';
+
+const require = createRequire(import.meta.url);
+const pkg = require('../package.json');
+
+const args = process.argv.slice(2);
+
+const usage = `ThreadShelf ${pkg.version} - local semantic search for your AI chats
+
+Usage:
+  npx threadshelf [port]            Start the web UI and API (default port 3000)
+
+Options:
+  -p, --port <port>   Port to listen on (default 3000, or $PORT)
+      --host <host>   Interface to bind (default 127.0.0.1, loopback only)
+      --data-dir <d>  Directory for persistent data
+                      (default: %LOCALAPPDATA%\ThreadShelf on Windows, ~/.threadshelf elsewhere)
+      --where         Print the resolved data and package directories, then exit
+  -v, --version       Print the version
+  -h, --help          Show this help
+
+Environment:
+  PORT, HOST, THREADSHELF_DATA_DIR, LANCEDB_PATH and the other documented
+  overrides keep working and take precedence over the defaults.
+`;
+
+let port = '';
+let showWhere = false;
+
+const takeValue = (flag, index) => {
+  const value = args[index + 1];
+  if (!value || value.startsWith('-')) {
+    console.error(`Missing value for ${flag}`);
+    process.exit(1);
+  }
+  return value;
+};
+
+for (let i = 0; i < args.length; i += 1) {
+  const arg = args[i];
+  if (arg === '-h' || arg === '--help') {
+    console.log(usage);
+    process.exit(0);
+  } else if (arg === '-v' || arg === '--version') {
+    console.log(pkg.version);
+    process.exit(0);
+  } else if (arg === '--where') {
+    showWhere = true;
+  } else if (arg === '-p' || arg === '--port') {
+    port = takeValue(arg, i);
+    i += 1;
+  } else if (arg === '--host') {
+    process.env.HOST = takeValue(arg, i);
+    i += 1;
+  } else if (arg === '--data-dir') {
+    process.env.THREADSHELF_DATA_DIR = takeValue(arg, i);
+    i += 1;
+  } else if (/^\d+$/.test(arg)) {
+    port = arg;
+  } else {
+    console.error(`Unknown argument: ${arg}\n\n${usage}`);
+    process.exit(1);
+  }
+}
+
+if (port) process.env.PORT = port;
+
+if (showWhere) {
+  const { dataDir, packageRoot } = await import('../dist/src/paths.js');
+  console.log('package :', packageRoot());
+  console.log('data    :', dataDir());
+  process.exit(0);
+}
+
+// server.ts reads process.argv[2] as a port; it is already normalised into
+// process.env.PORT above, so hide the raw argv from it.
+process.argv = [process.argv[0], process.argv[1]];
+
+await import('../dist/src/server.js');

package/dist/mcp/server.js

@@ -0,0 +1,388 @@
+#!/usr/bin/env tsx
+import { createInterface } from 'readline';
+import { searchAcrossCollections } from '../src/services/search.js';
+import { loadThread } from '../src/services/thread.js';
+import { getAllCollections } from '../src/services/collections.js';
+import { getStatsForCollection } from '../src/services/stats.js';
+import { listSourceFilesInCollection } from '../src/store.js';
+import { recoverPendingIndexes, startIndexRecovery } from '../src/store.js';
+import { ValidationError, normalizeCollectionSelector, normalizeCollectionName, normalizeQuery, normalizeCount, normalizeRoles, normalizeBoolean, normalizeOptionalString, normalizeDateRange, normalizeSearchMode, } from '../src/validation.js';
+import { createRequire } from 'node:module';
+import { packagePath } from '../src/paths.js';
+// Read at runtime rather than `import ... with { type: 'json' }`: a JSON import
+// makes tsc copy package.json into dist/, which would shadow the real package
+// root when resolving bundled assets.
+const pkg = createRequire(import.meta.url)(packagePath('package.json'));
+const DEFAULT_PROTOCOL_VERSION = '2024-11-05';
+const SUPPORTED_PROTOCOL_VERSIONS = ['2025-06-18', '2025-03-26', DEFAULT_PROTOCOL_VERSION];
+const SERVER_INFO = { name: 'threadshelf-mcp', version: pkg.version };
+const TOOL_DEFINITIONS = [
+    {
+        name: 'list_collections',
+        description: 'List all LanceDB collections discovered locally.',
+        inputSchema: { type: 'object', properties: {}, additionalProperties: false },
+    },
+    {
+        name: 'list_files',
+        description: 'List unique source files indexed inside a collection.',
+        inputSchema: {
+            type: 'object',
+            properties: {
+                collection: {
+                    type: 'string',
+                    description: 'Collection name. Use "all" to list across every collection. Defaults to "all".',
+                },
+            },
+            additionalProperties: false,
+        },
+    },
+    {
+        name: 'get_stats',
+        description: 'Return chunk, file, role, and per-collection stats for a collection or all collections.',
+        inputSchema: {
+            type: 'object',
+            properties: {
+                collection: {
+                    type: 'string',
+                    description: 'Collection name. Use "all" to aggregate every collection. Defaults to "all".',
+                },
+            },
+            additionalProperties: false,
+        },
+    },
+    {
+        name: 'search',
+        description: 'Semantic search over the chosen collection. Returns ranked text snippets with source metadata.',
+        inputSchema: {
+            type: 'object',
+            properties: {
+                query: { type: 'string', description: 'Natural-language query (any language).' },
+                collection: { type: 'string', description: 'Collection name or "all". Defaults to "all".' },
+                n: { type: 'integer', description: 'Max results (1-50). Default 15.' },
+                roles: {
+                    type: 'array',
+                    items: { type: 'string', enum: ['user', 'thinking', 'ai'] },
+                    description: 'Restrict to a subset of roles.',
+                },
+                keywordBoost: {
+                    type: 'boolean',
+                    description: 'When true, re-rank so chunks containing the exact query text appear first.',
+                },
+                mode: {
+                    type: 'string',
+                    enum: ['semantic', 'keyword'],
+                    description: 'Search mode. "semantic" (default) ranks by embedding similarity; "keyword" returns exact case-insensitive substring matches (best for identifiers, error strings, code).',
+                },
+                model: { type: 'string', description: 'Restrict results to models containing this text.' },
+                from: { type: 'string', description: 'Inclusive ISO date or YYYY-MM-DD lower bound.' },
+                to: { type: 'string', description: 'Inclusive ISO date or YYYY-MM-DD upper bound.' },
+            },
+            required: ['query'],
+            additionalProperties: false,
+        },
+    },
+    {
+        name: 'read_thread',
+        description: 'Load and parse a full export file, returning the normalized turn array.',
+        inputSchema: {
+            type: 'object',
+            properties: {
+                sourceFile: { type: 'string', description: 'Absolute path to an indexed export file.' },
+                collection: {
+                    type: 'string',
+                    description: 'Restrict the lookup to a specific collection (or "all"). Defaults to "all".',
+                },
+                conversationKey: {
+                    type: 'string',
+                    description: 'Conversation key returned by search for multi-conversation export files.',
+                },
+            },
+            required: ['sourceFile'],
+            additionalProperties: false,
+        },
+    },
+];
+const RESOURCE_TEMPLATES = [
+    {
+        uriTemplate: 'threadshelf://collections',
+        name: 'collections',
+        description: 'JSON list of all collections.',
+        mimeType: 'application/json',
+    },
+    {
+        uriTemplate: 'threadshelf://collections/{collection}/files',
+        name: 'collection-files',
+        description: 'JSON list of indexed source files within a collection.',
+        mimeType: 'application/json',
+    },
+    {
+        uriTemplate: 'threadshelf://thread?path={absolutePath}',
+        name: 'thread',
+        description: 'Parsed turn-by-turn thread for an indexed source file.',
+        mimeType: 'application/json',
+    },
+];
+const ERROR_CODES = {
+    parseError: -32700,
+    invalidRequest: -32600,
+    methodNotFound: -32601,
+    invalidParams: -32602,
+    internalError: -32603,
+};
+const jsonRpcResult = (id, result) => {
+    return JSON.stringify({ jsonrpc: '2.0', id, result });
+};
+const jsonRpcError = (id, code, message, data) => {
+    const error = { code, message };
+    if (data !== undefined)
+        error.data = data;
+    return JSON.stringify({ jsonrpc: '2.0', id, error });
+};
+const writeMessage = (payload) => {
+    process.stdout.write(`${payload}\n`);
+};
+const asTextContent = (value) => {
+    const text = typeof value === 'string' ? value : JSON.stringify(value, null, 2);
+    return [{ type: 'text', text }];
+};
+const toolListFiles = async (args = {}) => {
+    const collection = normalizeCollectionSelector(args.collection, { defaultValue: 'all' });
+    if (collection === 'all') {
+        const collections = await getAllCollections();
+        return Promise.all(collections.map(async (name) => ({
+            collection: name,
+            files: await listSourceFilesInCollection(name),
+        })));
+    }
+    const files = await listSourceFilesInCollection(collection);
+    return { collection, files };
+};
+const toolGetStats = async (args = {}) => {
+    const collection = normalizeCollectionSelector(args.collection, { defaultValue: 'all' });
+    return getStatsForCollection(collection);
+};
+const toolSearch = async (args = {}) => {
+    const query = normalizeQuery(args.query, { field: 'query' });
+    const collection = normalizeCollectionSelector(args.collection, { defaultValue: 'all' });
+    const n = normalizeCount(args.n, { defaultValue: 15 });
+    const roles = normalizeRoles(args.roles);
+    const keywordBoost = normalizeBoolean(args.keywordBoost, { field: 'keywordBoost' });
+    const model = normalizeOptionalString(args.model, { field: 'model' });
+    const { from, to } = normalizeDateRange(args.from, args.to);
+    const mode = normalizeSearchMode(args.mode);
+    // Bounded catch-up; failed jobs keep their backoff and never re-embed per query.
+    await recoverPendingIndexes({ collection });
+    const results = await searchAcrossCollections(query, collection, {
+        n,
+        roles: roles ?? undefined,
+        keywordBoost,
+        model,
+        from,
+        to,
+        mode,
+    });
+    return { collection, results };
+};
+const toolReadThread = async (args = {}) => {
+    if (!args || typeof args.sourceFile !== 'string' || !args.sourceFile.trim()) {
+        throw new ValidationError('Missing sourceFile', { field: 'sourceFile' });
+    }
+    const sourceFile = args.sourceFile;
+    if (sourceFile.length > 4096 || sourceFile.includes('\0')) {
+        throw new ValidationError('Invalid sourceFile', { field: 'sourceFile' });
+    }
+    const collection = normalizeCollectionSelector(args.collection, { defaultValue: 'all' });
+    let conversationKey;
+    if (args.conversationKey !== undefined && args.conversationKey !== null) {
+        if (typeof args.conversationKey !== 'string' ||
+            args.conversationKey.length > 4096 ||
+            args.conversationKey.includes('\0')) {
+            throw new ValidationError('Invalid conversationKey', { field: 'conversationKey' });
+        }
+        conversationKey = args.conversationKey || undefined;
+    }
+    return loadThread(sourceFile, collection, conversationKey);
+};
+const parseResourceUri = (uri) => {
+    const match = /^threadshelf:\/\/(.+)$/.exec(uri || '');
+    if (!match)
+        return null;
+    const rest = match[1];
+    if (rest === 'collections')
+        return { kind: 'collections' };
+    const filesMatch = /^collections\/([^/]+)\/files$/.exec(rest);
+    if (filesMatch)
+        return { kind: 'files', collection: decodeURIComponent(filesMatch[1]) };
+    if (rest.startsWith('thread')) {
+        const query = rest.split('?')[1] || '';
+        const params = new URLSearchParams(query);
+        const path = params.get('path');
+        if (!path)
+            return null;
+        return { kind: 'thread', path };
+    }
+    return null;
+};
+const readResource = async (uri) => {
+    const parsed = parseResourceUri(uri);
+    if (!parsed)
+        throw new ValidationError(`Unknown resource URI: ${uri}`, { field: 'uri' });
+    if (parsed.kind === 'collections') {
+        const collections = await getAllCollections();
+        return { uri, mimeType: 'application/json', text: JSON.stringify(collections) };
+    }
+    if (parsed.kind === 'files') {
+        const collection = normalizeCollectionName(parsed.collection);
+        const files = await listSourceFilesInCollection(collection);
+        return { uri, mimeType: 'application/json', text: JSON.stringify({ collection, files }) };
+    }
+    if (parsed.kind === 'thread') {
+        const thread = await loadThread(parsed.path, 'all');
+        return { uri, mimeType: 'application/json', text: JSON.stringify(thread) };
+    }
+    throw new ValidationError('Unsupported resource', { field: 'uri' });
+};
+const HANDLERS = {
+    async initialize(params = {}) {
+        const requestedVersion = typeof params.protocolVersion === 'string' ? params.protocolVersion : undefined;
+        const protocolVersion = SUPPORTED_PROTOCOL_VERSIONS.includes(requestedVersion)
+            ? requestedVersion
+            : DEFAULT_PROTOCOL_VERSION;
+        return {
+            protocolVersion,
+            capabilities: {
+                tools: { listChanged: false },
+                resources: { subscribe: false, listChanged: false },
+            },
+            serverInfo: SERVER_INFO,
+        };
+    },
+    async 'tools/list'() {
+        return { tools: TOOL_DEFINITIONS };
+    },
+    async 'tools/call'(params) {
+        const { name, arguments: args } = params;
+        try {
+            let payload;
+            switch (name) {
+                case 'list_collections':
+                    payload = await getAllCollections();
+                    break;
+                case 'list_files':
+                    payload = await toolListFiles(args ?? {});
+                    break;
+                case 'get_stats':
+                    payload = await toolGetStats(args ?? {});
+                    break;
+                case 'search':
+                    payload = await toolSearch(args ?? {});
+                    break;
+                case 'read_thread':
+                    payload = await toolReadThread(args ?? {});
+                    break;
+                default:
+                    throw new ValidationError(`Unknown tool: ${name}`, { field: 'name' });
+            }
+            return { content: asTextContent(payload), isError: false };
+        }
+        catch (e) {
+            const message = e instanceof ValidationError ? e.message : `Tool failed: ${e.message}`;
+            return { content: asTextContent({ error: message }), isError: true };
+        }
+    },
+    async 'resources/list'() {
+        return { resources: [], resourceTemplates: RESOURCE_TEMPLATES };
+    },
+    async 'resources/read'(params) {
+        const { uri } = params;
+        try {
+            const contents = await readResource(uri);
+            return { contents: [contents] };
+        }
+        catch (e) {
+            throw e instanceof ValidationError
+                ? e
+                : new ValidationError(`Resource read failed: ${e.message}`, { field: 'uri' });
+        }
+    },
+    async 'notifications/initialized'() {
+        return null;
+    },
+    async ping() {
+        return {};
+    },
+};
+const handleMessage = async (message) => {
+    if (message.jsonrpc !== '2.0') {
+        if (message.id !== undefined) {
+            writeMessage(jsonRpcError(message.id, ERROR_CODES.invalidRequest, 'Expected jsonrpc 2.0'));
+        }
+        return;
+    }
+    const handler = message.method ? HANDLERS[message.method] : undefined;
+    const isNotification = message.id === undefined;
+    if (!handler) {
+        if (!isNotification) {
+            writeMessage(jsonRpcError(message.id, ERROR_CODES.methodNotFound, `Method not found: ${message.method}`));
+        }
+        return;
+    }
+    try {
+        const result = await handler(message.params ?? {});
+        if (!isNotification) {
+            writeMessage(jsonRpcResult(message.id, result ?? {}));
+        }
+    }
+    catch (e) {
+        if (isNotification)
+            return;
+        const code = e instanceof ValidationError ? ERROR_CODES.invalidParams : ERROR_CODES.internalError;
+        writeMessage(jsonRpcError(message.id, code, e.message));
+    }
+};
+export const runServer = ({ input = process.stdin } = {}) => {
+    const rl = createInterface({ input, terminal: false });
+    rl.on('line', (raw) => {
+        const line = raw.trim();
+        if (!line)
+            return;
+        let message;
+        try {
+            message = JSON.parse(line);
+        }
+        catch {
+            writeMessage(jsonRpcError(null, ERROR_CODES.parseError, 'Invalid JSON in request'));
+            return;
+        }
+        handleMessage(message).catch((e) => {
+            writeMessage(jsonRpcError(message.id ?? null, ERROR_CODES.internalError, e.message));
+        });
+    });
+    return { close: () => rl.close() };
+};
+export const __testing__ = {
+    HANDLERS,
+    TOOL_DEFINITIONS,
+    RESOURCE_TEMPLATES,
+    parseResourceUri,
+    toolListFiles,
+    toolGetStats,
+    toolSearch,
+    toolReadThread,
+    readResource,
+    handleMessage,
+};
+const isEntrypoint = (() => {
+    try {
+        const argvUrl = new URL(`file://${process.argv[1].replace(/\\/g, '/')}`).href;
+        return import.meta.url === argvUrl;
+    }
+    catch {
+        return false;
+    }
+})();
+if (isEntrypoint) {
+    const stopRecovery = startIndexRecovery();
+    process.stdin.once('end', stopRecovery);
+    runServer();
+}

package/dist/src/chunking.js

@@ -0,0 +1,72 @@
+const MAX_CHUNK_CHARS = Math.max(200, Number(process.env.CHUNK_MAX_CHARS) || 2000);
+const OVERLAP_CHARS = Math.max(0, Number(process.env.CHUNK_OVERLAP_CHARS) || 100);
+export const chunkTurns = (turns, meta) => {
+    const chunks = [];
+    let turnIndex = 0;
+    for (const turn of turns) {
+        const role = turn.user !== undefined ? 'user' : turn.thinking !== undefined ? 'thinking' : 'ai';
+        const text = turn.user ?? turn.thinking ?? turn.ai ?? '';
+        if (!isIndexableText(text)) {
+            turnIndex++;
+            continue;
+        }
+        const parts = splitText(text, MAX_CHUNK_CHARS, OVERLAP_CHARS);
+        for (const part of parts) {
+            chunks.push({
+                text: part,
+                role,
+                turnIndex,
+                sourceFile: meta.sourceFile,
+                provider: meta.provider,
+                conversationKey: meta.conversationKey,
+                title: meta.title,
+                model: turn.model,
+                createdAt: turn.createdAt,
+                createdInThreadShelf: turn.createdInThreadShelf,
+                generationProvider: turn.generationProvider,
+            });
+        }
+        turnIndex++;
+    }
+    return chunks;
+};
+export const isIndexableText = (text) => {
+    if (!text || text === '[image]')
+        return false;
+    const trimmed = text.trim();
+    if (!trimmed)
+        return false;
+    if (trimmed === '{}' || trimmed === '[]' || trimmed === '<')
+        return false;
+    if (/^search\(/i.test(trimmed))
+        return false;
+    if (trimmed.length < 2)
+        return false;
+    return true;
+};
+const splitText = (text, maxChars, overlap) => {
+    if (text.length <= maxChars)
+        return [text];
+    const parts = [];
+    let start = 0;
+    while (start < text.length) {
+        let end = Math.min(start + maxChars, text.length);
+        if (end < text.length) {
+            const breakAt = text.lastIndexOf('\n\n', end);
+            if (breakAt > start) {
+                end = breakAt + 2;
+            }
+            else {
+                const lineBreak = text.lastIndexOf('\n', end);
+                if (lineBreak > start)
+                    end = lineBreak + 1;
+            }
+        }
+        const chunk = text.slice(start, end).trim();
+        if (chunk.length > 0)
+            parts.push(chunk);
+        const nextStart = end - (end < text.length ? overlap : 0);
+        start = nextStart > start ? nextStart : end;
+    }
+    return parts;
+};

package/dist/src/cli.js

@@ -0,0 +1,24 @@
+#!/usr/bin/env tsx
+import { parseFile } from './parser.js';
+const file = process.argv[2];
+if (!file) {
+    console.error('Usage: npm run parse -- <file> -- [--no-user] [--no-thinking] [--no-ai]');
+    process.exit(1);
+}
+const options = {
+    includeUser: !process.argv.includes('--no-user'),
+    includeThinking: !process.argv.includes('--no-thinking'),
+    includeAi: !process.argv.includes('--no-ai'),
+};
+try {
+    const result = await parseFile(file, options);
+    if (result.error) {
+        console.error(result.error);
+        process.exit(1);
+    }
+    console.log(JSON.stringify(result.turns, null, 2));
+}
+catch (e) {
+    console.error(e.message);
+    process.exit(1);
+}