threadshelf 1.2.0

package/dist/src/search-cli.js

@@ -0,0 +1,93 @@
+#!/usr/bin/env tsx
+import { searchAcrossCollections } from './services/search.js';
+import { recoverPendingIndexes } from './store.js';
+import { normalizeCollectionSelector, normalizeCount, normalizeDateRange, normalizeOptionalString, normalizeQuery, normalizeRoles, normalizeSearchMode, } from './validation.js';
+const USAGE = `Usage: npm run search -- "<query>" -- [options]
+
+Options:
+  --collection <name>   Collection to search, or "all" (default: all)
+  --mode <mode>         "semantic" (default) or "keyword" (exact substring)
+  --roles <list>        Comma list of user,thinking,ai
+  --model <text>        Only results whose model contains this text
+  --from <date>         Inclusive ISO/YYYY-MM-DD lower bound
+  --to <date>           Inclusive ISO/YYYY-MM-DD upper bound
+  --n <count>           Max results, 1-50 (default: 10)
+  --json                Print raw JSON instead of readable output`;
+const args = process.argv.slice(2);
+const flags = new Map();
+const positional = [];
+let json = false;
+for (let i = 0; i < args.length; i++) {
+    const arg = args[i];
+    if (arg === '--') {
+        continue;
+    }
+    else if (arg === '--json') {
+        json = true;
+    }
+    else if (arg.startsWith('--')) {
+        const value = args[i + 1];
+        if (value === undefined || value.startsWith('--')) {
+            console.error(`Missing value for ${arg}\n\n${USAGE}`);
+            process.exit(1);
+        }
+        flags.set(arg.slice(2), value);
+        i++;
+    }
+    else {
+        positional.push(arg);
+    }
+}
+if (positional.length !== 1) {
+    console.error(USAGE);
+    process.exit(1);
+}
+try {
+    const query = normalizeQuery(positional[0], { field: 'query' });
+    const collection = normalizeCollectionSelector(flags.get('collection'), {
+        defaultValue: 'all',
+    });
+    const mode = normalizeSearchMode(flags.get('mode'));
+    const roles = normalizeRoles(flags.get('roles'));
+    const model = normalizeOptionalString(flags.get('model'), { field: 'model' });
+    const { from, to } = normalizeDateRange(flags.get('from'), flags.get('to'));
+    const n = normalizeCount(flags.get('n'), { defaultValue: 10 });
+    await recoverPendingIndexes({ collection });
+    const results = await searchAcrossCollections(query, collection, {
+        n,
+        mode,
+        roles: roles ?? undefined,
+        model,
+        from,
+        to,
+        keywordBoost: mode === 'semantic',
+    });
+    if (json) {
+        console.log(JSON.stringify({ query, collection, mode, results }, null, 2));
+        process.exit(0);
+    }
+    if (results.length === 0) {
+        console.log(`No results for "${query}" in ${collection}.`);
+        process.exit(0);
+    }
+    for (const [index, result] of results.entries()) {
+        const meta = result.metadata;
+        const score = result.distance != null ? ` score=${(1 - result.distance).toFixed(3)}` : '';
+        const modelLabel = meta.model ? ` model=${meta.model}` : '';
+        const dateLabel = meta.createdAt ? ` date=${meta.createdAt.slice(0, 10)}` : '';
+        const collectionLabel = meta.collection ? `${meta.collection} · ` : '';
+        const snippet = result.document.replace(/\s+/g, ' ').trim();
+        const preview = snippet.length > 240 ? `${snippet.slice(0, 237)}...` : snippet;
+        console.log(`${index + 1}. [${meta.role}]${score}${modelLabel}${dateLabel}`);
+        console.log(`   ${collectionLabel}${meta.sourceFile}`);
+        if (meta.title)
+            console.log(`   "${meta.title}"`);
+        console.log(`   ${preview}`);
+        console.log('');
+    }
+    process.exit(0);
+}
+catch (e) {
+    console.error(`[search] ${e.message}`);
+    process.exit(1);
+}

package/dist/src/server.js

@@ -0,0 +1,78 @@
+import './env.js';
+import express from 'express';
+import { join } from 'path';
+import apiRouter from './routes/index.js';
+import { dataDir, packagePath } from './paths.js';
+import { startIndexRecovery } from './store.js';
+const app = express();
+app.use(express.json({ limit: '512kb' }));
+// Package assets resolve against the installed module, never process.cwd():
+// `npx threadshelf` runs from whatever directory the user happens to be in.
+const PUBLIC = packagePath('public');
+app.use(express.static(PUBLIC));
+// Serve the browser-console export scripts so the UI can offer a "copy script"
+// button (e.g. the OpenRouter exporter). Read-only static files.
+app.use('/scripts', express.static(packagePath('scripts')));
+const normalizeRequestHost = (value) => {
+    const host = String(value || '')
+        .trim()
+        .toLowerCase();
+    if (!host)
+        return '';
+    if (host.startsWith('['))
+        return host.replace(/]:\d+$/, ']').replace(/^\[(.*)]$/, '$1');
+    return host.replace(/:\d+$/, '');
+};
+const localHosts = () => {
+    const configuredHost = normalizeRequestHost(process.env.HOST || '127.0.0.1');
+    const configuredAllowedHosts = (process.env.ALLOWED_HOSTS || '')
+        .split(',')
+        .map((host) => normalizeRequestHost(host))
+        .filter(Boolean);
+    return new Set(['localhost', '127.0.0.1', '::1', configuredHost, ...configuredAllowedHosts].filter((host) => host && host !== '0.0.0.0' && host !== '::'));
+};
+app.use('/api', (req, res, next) => {
+    const allowedHosts = localHosts();
+    const host = normalizeRequestHost(req.headers.host);
+    if (!allowedHosts.has(host)) {
+        return res.status(403).json({ error: 'Forbidden host' });
+    }
+    const origin = req.headers.origin;
+    if (origin) {
+        try {
+            const originHost = normalizeRequestHost(new URL(origin).host);
+            if (!allowedHosts.has(originHost)) {
+                return res.status(403).json({ error: 'Forbidden origin' });
+            }
+        }
+        catch {
+            return res.status(403).json({ error: 'Invalid origin' });
+        }
+    }
+    next();
+});
+app.use(apiRouter);
+app.use('/api', (_req, res) => {
+    res.status(404).json({ error: 'Not found' });
+});
+app.get(/.*/, (_req, res, next) => {
+    res.sendFile(join(PUBLIC, 'index.html'), (err) => {
+        if (err)
+            next(err);
+    });
+});
+app.use((err, _req, res, _next) => {
+    console.error('Express error', err);
+    if (err.name === 'MulterError') {
+        return res.status(400).json({ error: err.message });
+    }
+    res.status(500).json({ error: err?.message || 'Server error' });
+});
+const portArg = process.argv[2];
+const PORT = Number(portArg ?? process.env.PORT) || 3000;
+const HOST = process.env.HOST || '127.0.0.1';
+app.listen(PORT, HOST, () => {
+    startIndexRecovery();
+    console.log(`Server: http://localhost:${PORT}`);
+    console.log('Data directory:', dataDir());
+});

package/dist/src/services/collections.js

@@ -0,0 +1,58 @@
+import { readFile, writeFile } from 'fs/promises';
+import { join } from 'path';
+import { dataPath } from '../paths.js';
+import { listCollections, dropCollection } from '../store.js';
+// Overridable so test servers with an isolated LanceDB do not share (and
+// pollute) the real registry file in the repo/app directory.
+const COLLECTIONS_FILE = process.env.COLLECTIONS_PATH || dataPath('collections');
+export const readManualCollections = async () => {
+    try {
+        const raw = await readFile(COLLECTIONS_FILE, 'utf-8');
+        const data = JSON.parse(raw);
+        return Array.isArray(data.collections) ? data.collections : [];
+    }
+    catch {
+        return [];
+    }
+};
+export const writeManualCollections = async (collections) => {
+    await writeFile(COLLECTIONS_FILE, JSON.stringify({ collections }, null, 2));
+};
+export const getAllCollections = async () => {
+    const [tableCollections, manualCollections] = await Promise.all([
+        listCollections(),
+        readManualCollections(),
+    ]);
+    return [...new Set([...tableCollections, ...manualCollections, 'chunks'])].sort();
+};
+export const addManualCollection = async (name) => {
+    const collections = await readManualCollections();
+    if (!collections.includes(name)) {
+        collections.push(name);
+        collections.sort();
+        await writeManualCollections(collections);
+    }
+};
+export const removeManualCollection = async (name) => {
+    const collections = await readManualCollections();
+    const filtered = collections.filter((c) => c !== name);
+    if (filtered.length !== collections.length) {
+        await writeManualCollections(filtered);
+    }
+};
+export const ensureCollectionExists = async (name) => {
+    const all = await getAllCollections();
+    if (!all.includes(name)) {
+        await addManualCollection(name);
+    }
+};
+export const deleteCollectionFull = async (name, uploadsDir) => {
+    const { existsSync } = await import('fs');
+    const { rm } = await import('fs/promises');
+    await dropCollection(name);
+    await removeManualCollection(name);
+    const uploadDir = join(uploadsDir, name);
+    if (existsSync(uploadDir)) {
+        await rm(uploadDir, { recursive: true, force: true });
+    }
+};

package/dist/src/services/insights.js

@@ -0,0 +1,111 @@
+import { getCollectionStats, listThreadSummaries, scanChunkMeta, } from '../store.js';
+import { getAllCollections } from './collections.js';
+const TOP_MODELS = 10;
+const TOP_THREADS = 10;
+const normalizeModelName = (model) => model.replace(/^models\//, '').trim();
+const monthOf = (iso) => {
+    // createdAt is a normalized ISO string; anything shorter is unusable.
+    return /^\d{4}-\d{2}/.test(iso) ? iso.slice(0, 7) : null;
+};
+// Fill gaps so the chart shows a continuous timeline (quiet months at zero).
+const fillMonthGaps = (buckets) => {
+    const months = [...buckets.keys()].sort();
+    if (months.length === 0)
+        return [];
+    const [first] = months;
+    const last = months[months.length - 1];
+    const out = [];
+    let [year, month] = first.split('-').map(Number);
+    for (;;) {
+        const key = `${year}-${String(month).padStart(2, '0')}`;
+        out.push({ month: key, count: buckets.get(key) ?? 0 });
+        if (key === last || out.length > 600)
+            break;
+        month++;
+        if (month > 12) {
+            month = 1;
+            year++;
+        }
+    }
+    return out;
+};
+export const getInsights = async (collection) => {
+    const collections = collection === 'all' ? await getAllCollections() : [collection];
+    const [statsList, summariesList, chunkMetaList] = await Promise.all([
+        Promise.all(collections.map((name) => getCollectionStats(name))),
+        Promise.all(collections.map((name) => listThreadSummaries(name))),
+        Promise.all(collections.map((name) => scanChunkMeta(name))),
+    ]);
+    const totals = { collections: 0, files: 0, chunks: 0, conversations: 0, turns: 0 };
+    const roles = { user: 0, thinking: 0, ai: 0 };
+    for (const stats of statsList) {
+        totals.files += stats.files;
+        totals.chunks += stats.chunks;
+        roles.user += stats.roles.user;
+        roles.thinking += stats.roles.thinking;
+        roles.ai += stats.roles.ai;
+        if (!stats.isEmpty)
+            totals.collections++;
+    }
+    const providerMap = new Map();
+    const longest = [];
+    for (let i = 0; i < collections.length; i++) {
+        const summaries = summariesList[i] ?? [];
+        totals.conversations += summaries.length;
+        for (const row of summaries) {
+            totals.turns += row.turnCount;
+            const provider = row.provider || 'unknown';
+            const entry = providerMap.get(provider) ?? { conversations: 0, turns: 0 };
+            entry.conversations++;
+            entry.turns += row.turnCount;
+            providerMap.set(provider, entry);
+            longest.push({
+                collection: collections[i],
+                sourceFile: row.sourceFile,
+                conversationKey: row.conversationKey,
+                title: row.title,
+                provider: row.provider,
+                turnCount: row.turnCount,
+                lastTurnAt: row.lastTurnAt,
+            });
+        }
+    }
+    longest.sort((a, b) => b.turnCount - a.turnCount);
+    const activityBuckets = new Map();
+    const modelCounts = new Map();
+    let firstActivity = null;
+    let lastActivity = null;
+    for (const rows of chunkMetaList) {
+        for (const row of rows) {
+            const month = monthOf(row.createdAt);
+            if (month) {
+                activityBuckets.set(month, (activityBuckets.get(month) ?? 0) + 1);
+                if (!firstActivity || row.createdAt < firstActivity)
+                    firstActivity = row.createdAt;
+                if (!lastActivity || row.createdAt > lastActivity)
+                    lastActivity = row.createdAt;
+            }
+            const model = normalizeModelName(row.model);
+            if (model)
+                modelCounts.set(model, (modelCounts.get(model) ?? 0) + 1);
+        }
+    }
+    const topModels = [...modelCounts.entries()]
+        .map(([model, count]) => ({ model, count }))
+        .sort((a, b) => b.count - a.count)
+        .slice(0, TOP_MODELS);
+    const providers = [...providerMap.entries()]
+        .map(([provider, entry]) => ({ provider, ...entry }))
+        .sort((a, b) => b.turns - a.turns);
+    return {
+        collection,
+        totals,
+        roles,
+        activity: fillMonthGaps(activityBuckets),
+        topModels,
+        providers,
+        longestThreads: longest.slice(0, TOP_THREADS),
+        firstActivity,
+        lastActivity,
+    };
+};

package/dist/src/services/search.js

@@ -0,0 +1,68 @@
+import { searchCollection, keywordSearchCollection, keywordResultComparator, } from '../store.js';
+import { embedOne } from '../embedding.js';
+import { getAllCollections } from './collections.js';
+export const searchAcrossCollections = async (query, collection, opts = {}) => {
+    const n = opts.n ?? 15;
+    const model = opts.model;
+    const searchLimit = model ? 50 : n;
+    if (opts.mode === 'keyword') {
+        return keywordSearchAcrossCollections(query, collection, { ...opts, n });
+    }
+    // Generate the query vector once. "All collections" used to repeat the
+    // exact same model inference for every collection, multiplying latency as a
+    // user's archive grew.
+    const queryEmbedding = await embedOne(query);
+    if (collection === 'all') {
+        const collections = await getAllCollections();
+        const merged = await Promise.all(collections.map(async (name) => {
+            const rows = await searchCollection(name, query, { ...opts, n: searchLimit }, queryEmbedding);
+            return rows.map((row) => ({
+                ...row,
+                metadata: { ...row.metadata, collection: name },
+            }));
+        }));
+        let results = merged.flat();
+        results = filterResultsByModel(results, model);
+        results.sort(searchResultComparator(query, opts.keywordBoost === true));
+        return results.slice(0, n);
+    }
+    const rawResults = await searchCollection(collection, query, { ...opts, n: searchLimit }, queryEmbedding);
+    return filterResultsByModel(rawResults, model).slice(0, n);
+};
+// Exact substring search. Model/date/role filtering happens inside
+// keywordSearchCollection; here we only fan out and merge-rank.
+const keywordSearchAcrossCollections = async (query, collection, opts) => {
+    const n = opts.n ?? 15;
+    if (collection !== 'all') {
+        return keywordSearchCollection(collection, query, opts);
+    }
+    const collections = await getAllCollections();
+    const merged = await Promise.all(collections.map(async (name) => {
+        const rows = await keywordSearchCollection(name, query, opts);
+        return rows.map((row) => ({
+            ...row,
+            metadata: { ...row.metadata, collection: name },
+        }));
+    }));
+    const results = merged.flat();
+    results.sort(keywordResultComparator(query));
+    return results.slice(0, n);
+};
+export const searchResultComparator = (query, keywordBoost) => (a, b) => {
+    if (keywordBoost) {
+        const needle = query.trim().toLowerCase();
+        const aHas = needle.length > 0 && a.document.toLowerCase().includes(needle);
+        const bHas = needle.length > 0 && b.document.toLowerCase().includes(needle);
+        if (aHas !== bHas)
+            return aHas ? -1 : 1;
+    }
+    return (a.distance ?? Number.POSITIVE_INFINITY) - (b.distance ?? Number.POSITIVE_INFINITY);
+};
+const filterResultsByModel = (results, model) => {
+    if (!model)
+        return results;
+    const needle = model.toLowerCase();
+    return results.filter((row) => String(row.metadata.model || '')
+        .toLowerCase()
+        .includes(needle));
+};

package/dist/src/services/stats.js

@@ -0,0 +1,35 @@
+import { getCollectionStats } from '../store.js';
+import { getAllCollections } from './collections.js';
+export const getAllCollectionStats = async () => {
+    const collections = await getAllCollections();
+    const stats = await Promise.all(collections.map((name) => getCollectionStats(name)));
+    const totals = {
+        files: 0,
+        conversations: 0,
+        chunks: 0,
+        roles: { user: 0, thinking: 0, ai: 0 },
+        indexedCollections: 0,
+    };
+    for (const item of stats) {
+        totals.files += item.files;
+        totals.conversations += item.conversations;
+        totals.chunks += item.chunks;
+        totals.roles.user += item.roles.user;
+        totals.roles.thinking += item.roles.thinking;
+        totals.roles.ai += item.roles.ai;
+        if (!item.isEmpty)
+            totals.indexedCollections++;
+    }
+    return {
+        collection: 'all',
+        ...totals,
+        totalCollections: collections.length,
+        isEmpty: totals.chunks === 0,
+        collections: stats,
+    };
+};
+export const getStatsForCollection = async (collection) => {
+    if (collection === 'all')
+        return getAllCollectionStats();
+    return getCollectionStats(collection);
+};

package/dist/src/services/thread.js

@@ -0,0 +1,140 @@
+import { readFile } from 'fs/promises';
+import { parseExport, getConversationFromExport } from '../parser.js';
+import { getStoredThreads, listSourceFilesInCollection } from '../store.js';
+import { validateTurns } from '../validation.js';
+import { getAllCollections } from './collections.js';
+import { portableModelLabel } from '../model-label.js';
+const sanitizeTurns = (turns) => {
+    const portable = turns.map((turn) => turn.model ? { ...turn, model: portableModelLabel(turn.model) } : turn);
+    try {
+        validateTurns(portable);
+        return portable;
+    }
+    catch (e) {
+        console.warn('[thread] dropped invalid turns:', e.message);
+        return portable.filter((turn) => turn &&
+            (typeof turn.user === 'string' ||
+                typeof turn.thinking === 'string' ||
+                typeof turn.ai === 'string'));
+    }
+};
+const parseStoredTurns = (turnsJson) => {
+    try {
+        const parsed = JSON.parse(turnsJson);
+        return Array.isArray(parsed) ? parsed : [];
+    }
+    catch {
+        return [];
+    }
+};
+// With collection "all" the same source file can be stored once per collection.
+// Keep the most recently ingested copy so the thread matches the freshest index.
+const pickNewestGroup = (rows) => {
+    const byCollection = new Map();
+    for (const row of rows) {
+        const group = byCollection.get(row.collection) ?? [];
+        group.push(row);
+        byCollection.set(row.collection, group);
+    }
+    let newest = [];
+    let newestAt = '';
+    for (const group of byCollection.values()) {
+        const at = group[0]?.ingestedAt ?? '';
+        if (!newest.length || at > newestAt) {
+            newest = group;
+            newestAt = at;
+        }
+    }
+    return newest;
+};
+const loadStoredThread = async (sourceFile, collection, conversationKey) => {
+    const rows = await getStoredThreads(collection === 'all' ? null : collection, sourceFile);
+    if (!rows.length)
+        return null;
+    const stored = pickNewestGroup(rows);
+    if (conversationKey) {
+        const row = stored.find((entry) => entry.conversationKey === conversationKey);
+        // A stale link (e.g. an old URL after re-indexing a rewritten export) falls
+        // back to re-parsing the source file below.
+        if (!row)
+            return null;
+        return {
+            sourceFile,
+            conversationKey: row.conversationKey,
+            title: row.title || undefined,
+            createdInThreadShelf: row.createdInThreadShelf,
+            threadCreatedAt: row.threadCreatedAt || undefined,
+            turns: sanitizeTurns(parseStoredTurns(row.turnsJson)),
+        };
+    }
+    if (stored.length === 1) {
+        const row = stored[0];
+        return {
+            sourceFile,
+            conversationKey: row.conversationKey,
+            title: row.title || undefined,
+            createdInThreadShelf: row.createdInThreadShelf,
+            threadCreatedAt: row.threadCreatedAt || undefined,
+            turns: sanitizeTurns(parseStoredTurns(row.turnsJson)),
+        };
+    }
+    // No key against a multi-conversation export: flatten in original file order,
+    // mirroring the legacy parseExport behaviour.
+    return {
+        sourceFile,
+        conversationKey: '',
+        createdInThreadShelf: stored.every((row) => row.createdInThreadShelf),
+        threadCreatedAt: stored.find((row) => row.threadCreatedAt)?.threadCreatedAt,
+        turns: sanitizeTurns(stored.flatMap((row) => parseStoredTurns(row.turnsJson))),
+    };
+};
+const loadThreadFromSourceFile = async (sourceFile, collection, conversationKey) => {
+    const allowedFiles = collection === 'all'
+        ? (await Promise.all((await getAllCollections()).map((n) => listSourceFilesInCollection(n)))).flat()
+        : await listSourceFilesInCollection(collection);
+    if (!allowedFiles.includes(sourceFile)) {
+        throw new NotFoundError('File not in collection or not found');
+    }
+    let raw;
+    try {
+        raw = await readFile(sourceFile, 'utf-8');
+    }
+    catch {
+        throw new NotFoundError('Source file is no longer readable and this collection predates stored threads — re-index to restore thread view');
+    }
+    const parsed = conversationKey
+        ? getConversationFromExport(raw, conversationKey)
+        : { conversation: null, error: null };
+    if (parsed.error)
+        throw new NotFoundError(parsed.error);
+    const fallback = !conversationKey ? parseExport(raw) : null;
+    if (fallback?.error)
+        throw new BadRequestError(fallback.error);
+    const selected = parsed.conversation;
+    const turns = sanitizeTurns(selected?.turns || fallback?.turns || []);
+    return {
+        sourceFile,
+        conversationKey: selected?.key || conversationKey || '',
+        title: selected?.title,
+        createdInThreadShelf: false,
+        turns,
+    };
+};
+export const loadThread = async (sourceFile, collection, conversationKey) => {
+    const stored = await loadStoredThread(sourceFile, collection, conversationKey);
+    if (stored)
+        return stored;
+    return loadThreadFromSourceFile(sourceFile, collection, conversationKey);
+};
+export class NotFoundError extends Error {
+    constructor(message) {
+        super(message);
+        this.name = 'NotFoundError';
+    }
+}
+export class BadRequestError extends Error {
+    constructor(message) {
+        super(message);
+        this.name = 'BadRequestError';
+    }
+}