threadshelf 1.2.0

package/dist/src/paths.js

@@ -0,0 +1,79 @@
+import { existsSync } from 'node:fs';
+import { homedir } from 'node:os';
+import { basename, dirname, join, resolve } from 'node:path';
+import { fileURLToPath } from 'node:url';
+/**
+ * Single source of truth for "where does ThreadShelf read and write things".
+ *
+ * Two kinds of location, deliberately kept apart:
+ *
+ * - **Package files** (built UI, browser export scripts) live next to the
+ *   installed module and are resolved from `import.meta.url`. They must never
+ *   be resolved from `process.cwd()`: `npx threadshelf` runs with the user's
+ *   shell directory as cwd, which has nothing to do with the package.
+ * - **Persistent user data** (LanceDB, uploads, collections, generation config)
+ *   lives outside the package entirely. An npm/npx install directory is
+ *   disposable — npm may wipe its `_npx` cache at any time — so nothing the
+ *   user cares about may be stored there.
+ *
+ * Running from a repository checkout keeps the historical repo-local layout so
+ * development and the existing test harness are unaffected.
+ */
+const moduleDir = dirname(fileURLToPath(import.meta.url));
+/**
+ * This module lives at `<root>/src/paths.ts` in development and at
+ * `<root>/dist/src/paths.js` once compiled, so the package root is one or two
+ * levels up depending on which copy is running.
+ */
+const resolvePackageRoot = (dir) => {
+    const parent = dirname(dir);
+    return basename(parent) === 'dist' ? dirname(parent) : parent;
+};
+const PACKAGE_ROOT = resolvePackageRoot(moduleDir);
+/** Root of the installed package (or the repo in development). Static assets only. */
+export const packageRoot = () => PACKAGE_ROOT;
+/** Path to a file shipped inside the package, e.g. the built UI or an export script. */
+export const packagePath = (...segments) => join(PACKAGE_ROOT, ...segments);
+/**
+ * True when running from a source checkout rather than an installed package.
+ * The published tarball ships `dist/`, never `src/*.ts`, so this cannot be
+ * confused by an end user's install.
+ */
+export const isRepoCheckout = () => existsSync(join(PACKAGE_ROOT, 'src', 'server.ts'));
+/** Per-user data directory for an installed package. */
+export const userDataDir = (env = process.env, platform = process.platform, home = homedir()) => {
+    if (platform === 'win32') {
+        const localAppData = env.LOCALAPPDATA?.trim();
+        return join(localAppData || join(home, 'AppData', 'Local'), 'ThreadShelf');
+    }
+    return join(home, '.threadshelf');
+};
+const explicitDataDir = () => process.env.THREADSHELF_DATA_DIR?.trim() || '';
+/** Whether persistent files use the repo-local dotfile layout or the flat user-data layout. */
+const useRepoLayout = () => !explicitDataDir() && isRepoCheckout();
+/** Root directory for everything persistent. Never inside the package for an installed copy. */
+export const dataDir = () => resolve(explicitDataDir() || (isRepoCheckout() ? PACKAGE_ROOT : userDataDir()));
+/**
+ * Repo layout keeps the dotfile names the project has always used; the
+ * user-data layout drops the dots because the directory is already dedicated.
+ */
+const LAYOUT = {
+    lancedb: { repo: ['.lancedb'], user: ['lancedb'] },
+    uploads: { repo: ['.uploads'], user: ['uploads'] },
+    collections: { repo: ['.collections.json'], user: ['collections.json'] },
+    generationConfig: { repo: ['.threadshelf', 'generation.json'], user: ['generation.json'] },
+    masterPrompts: { repo: ['.threadshelf', 'master-prompts.json'], user: ['master-prompts.json'] },
+    generationErrorLog: {
+        repo: ['.threadshelf', 'generation-errors.log'],
+        user: ['generation-errors.log'],
+    },
+    models: { repo: ['.threadshelf', 'models'], user: ['models'] },
+    tools: { repo: ['.threadshelf', 'tools'], user: ['tools'] },
+    modelCache: { repo: ['.threadshelf', 'model-cache'], user: ['model-cache'] },
+    env: { repo: ['.env'], user: ['.env'] },
+};
+/** Absolute path of a persistent file or directory. Callers still apply their own env overrides. */
+export const dataPath = (key) => {
+    const entry = LAYOUT[key];
+    return resolve(dataDir(), ...(useRepoLayout() ? entry.repo : entry.user));
+};

package/dist/src/routes/collections.js

@@ -0,0 +1,97 @@
+import { Router } from 'express';
+import { dropCollection } from '../store.js';
+import { getAllCollections, addManualCollection, ensureCollectionExists, deleteCollectionFull, } from '../services/collections.js';
+import { getStatsForCollection } from '../services/stats.js';
+import { ValidationError, normalizeCollectionName, normalizeCollectionSelector, assertDeletableCollection, assertClearableCollection, } from '../validation.js';
+const router = Router();
+const sendValidationError = (res, err) => {
+    res.status(400).json({ error: err.message, field: err.field });
+};
+router.get('/api/collections', async (_req, res) => {
+    try {
+        const collections = await getAllCollections();
+        res.json({ collections });
+    }
+    catch (e) {
+        console.error('[/api/collections]', e);
+        res.status(500).json({ collections: ['chunks'] });
+    }
+});
+router.post('/api/collections', async (req, res) => {
+    let name;
+    try {
+        name = normalizeCollectionName(req.body?.name, { field: 'name' });
+    }
+    catch (e) {
+        if (e instanceof ValidationError)
+            return sendValidationError(res, e);
+        throw e;
+    }
+    try {
+        await addManualCollection(name);
+        res.json({ ok: true, collection: name });
+    }
+    catch (e) {
+        console.error('[/api/collections POST]', e);
+        res.status(500).json({ error: e.message });
+    }
+});
+router.post('/api/collections/:name/clear', async (req, res) => {
+    let name;
+    try {
+        name = assertClearableCollection(req.params.name);
+    }
+    catch (e) {
+        if (e instanceof ValidationError)
+            return sendValidationError(res, e);
+        throw e;
+    }
+    try {
+        await dropCollection(name);
+        await ensureCollectionExists(name);
+        res.json({ ok: true, collection: name });
+    }
+    catch (e) {
+        console.error('[/api/collections/:name/clear]', e);
+        res.status(500).json({ error: e.message });
+    }
+});
+router.delete('/api/collections/:name', async (req, res) => {
+    let name;
+    try {
+        name = assertDeletableCollection(req.params.name);
+    }
+    catch (e) {
+        if (e instanceof ValidationError)
+            return sendValidationError(res, e);
+        throw e;
+    }
+    try {
+        const uploadsDir = process.env.UPLOADS_DIR || '.uploads';
+        await deleteCollectionFull(name, uploadsDir);
+        res.json({ ok: true, collection: name });
+    }
+    catch (e) {
+        console.error('[/api/collections/:name DELETE]', e);
+        res.status(500).json({ error: e.message });
+    }
+});
+router.get('/api/collections/:name/stats', async (req, res) => {
+    let collection;
+    try {
+        collection = normalizeCollectionSelector(req.params.name);
+    }
+    catch (e) {
+        if (e instanceof ValidationError)
+            return sendValidationError(res, e);
+        throw e;
+    }
+    try {
+        res.json(await getStatsForCollection(collection));
+    }
+    catch (e) {
+        console.error('[/api/collections/:name/stats]', e);
+        res.status(500).json({ error: e.message });
+    }
+});
+export default router;

package/dist/src/routes/files.js

@@ -0,0 +1,136 @@
+import { Router } from 'express';
+import { readFile, stat } from 'fs/promises';
+import { basename } from 'path';
+import { listSourceFilesInCollection, listThreadSummaries, } from '../store.js';
+import { listConversationsFromExport } from '../parser.js';
+import { getAllCollections } from '../services/collections.js';
+import { ValidationError, normalizeCollectionSelector } from '../validation.js';
+const router = Router();
+// Parsed-export cache keyed by collection+file. Bounded so a long-running
+// server browsing many collections cannot grow it without limit; on overflow
+// the oldest half is evicted (Map preserves insertion order).
+const FILE_PARSE_CACHE_MAX = 512;
+const fileParseCache = new Map();
+const pruneFileParseCache = () => {
+    if (fileParseCache.size < FILE_PARSE_CACHE_MAX)
+        return;
+    const evict = Math.ceil(FILE_PARSE_CACHE_MAX / 2);
+    for (const key of [...fileParseCache.keys()].slice(0, evict)) {
+        fileParseCache.delete(key);
+    }
+};
+const expandFile = async (sourceFile, collection) => {
+    const info = await stat(sourceFile);
+    const cacheKey = `${collection}\0${sourceFile}`;
+    const cached = fileParseCache.get(cacheKey);
+    if (cached && cached.mtimeMs === info.mtimeMs) {
+        return cached.files;
+    }
+    const raw = await readFile(sourceFile, 'utf-8');
+    const parsed = listConversationsFromExport(raw);
+    if (parsed.error || !parsed.conversations.length) {
+        const fallback = [
+            {
+                sourceFile,
+                collection,
+                conversationKey: '',
+                title: basename(sourceFile),
+                turnCount: 0,
+            },
+        ];
+        pruneFileParseCache();
+        fileParseCache.set(cacheKey, { mtimeMs: info.mtimeMs, files: fallback });
+        return fallback;
+    }
+    const expanded = parsed.conversations.map((conversation) => ({
+        sourceFile,
+        collection,
+        conversationKey: conversation.key,
+        title: conversation.title,
+        turnCount: conversation.turnCount,
+    }));
+    pruneFileParseCache();
+    fileParseCache.set(cacheKey, { mtimeMs: info.mtimeMs, files: expanded });
+    return expanded;
+};
+// A source file may have been moved, rewritten, or deleted after indexing (e.g.
+// LM Studio rewrites its conversation files as you keep chatting). Never let one
+// unreadable file reject the whole listing — fall back to a minimal entry so the
+// rest of the conversations still show.
+const expandFileSafe = async (sourceFile, collection) => {
+    try {
+        return await expandFile(sourceFile, collection);
+    }
+    catch {
+        return [
+            {
+                sourceFile,
+                collection,
+                conversationKey: '',
+                title: basename(sourceFile),
+                turnCount: 0,
+            },
+        ];
+    }
+};
+// Conversations indexed since threads storage exists are listed straight from
+// the __threads table (no file I/O). Files indexed before that fall back to
+// re-parsing the export on disk.
+const listCollectionFiles = async (collection) => {
+    const [sourceFiles, summaries] = await Promise.all([
+        listSourceFilesInCollection(collection),
+        listThreadSummaries(collection),
+    ]);
+    const bySource = new Map();
+    for (const summary of summaries) {
+        const group = bySource.get(summary.sourceFile) ?? [];
+        group.push(summary);
+        bySource.set(summary.sourceFile, group);
+    }
+    // __threads is authoritative for normalized conversations. Chunk tables are
+    // only a search index and can legitimately lag behind (empty/new chats or a
+    // temporarily failed embedding), so never use them as the listing's root.
+    const knownSources = new Set([...sourceFiles, ...summaries.map((row) => row.sourceFile)]);
+    const expanded = await Promise.all([...knownSources].sort().map(async (sourceFile) => {
+        const rows = bySource.get(sourceFile);
+        if (rows?.length) {
+            return [...rows]
+                .sort((a, b) => a.ordinal - b.ordinal)
+                .map((row) => ({
+                sourceFile,
+                collection,
+                conversationKey: row.conversationKey,
+                title: row.title || basename(sourceFile),
+                turnCount: row.turnCount,
+                provider: row.provider || undefined,
+                lastTurnAt: row.lastTurnAt || undefined,
+                createdInThreadShelf: row.createdInThreadShelf,
+                hasThreadShelfTurns: row.hasThreadShelfTurns,
+            }));
+        }
+        return expandFileSafe(sourceFile, collection);
+    }));
+    return expanded.flat();
+};
+router.get('/api/files', async (req, res) => {
+    let collection;
+    try {
+        collection = normalizeCollectionSelector(req.query?.collection);
+    }
+    catch (e) {
+        if (e instanceof ValidationError) {
+            return res.status(400).json({ error: e.message, field: e.field });
+        }
+        throw e;
+    }
+    try {
+        const collections = collection === 'all' ? await getAllCollections() : [collection];
+        const files = (await Promise.all(collections.map(listCollectionFiles))).flat();
+        res.json({ files });
+    }
+    catch (e) {
+        console.error('[/api/files]', e);
+        res.status(500).json({ error: e.message });
+    }
+});
+export default router;