threadshelf 1.2.0

package/dist/src/ingest-cli.js

@@ -0,0 +1,95 @@
+#!/usr/bin/env tsx
+import { basename, resolve } from 'path';
+import { ingestFolder } from './ingest.js';
+import { watchFolder } from './watch.js';
+import { normalizeCollectionName } from './validation.js';
+import { recoverPendingIndexes, startIndexRecovery } from './store.js';
+const args = process.argv.slice(2);
+const clearFirst = args.includes('--clear');
+const watchMode = args.includes('--watch');
+const readNumberFlag = (name, fallback) => {
+    const index = args.indexOf(name);
+    if (index === -1)
+        return fallback;
+    const value = Number(args[index + 1]);
+    if (!Number.isFinite(value) || value < 0) {
+        console.error(`${name} expects a non-negative number of milliseconds`);
+        process.exit(1);
+    }
+    return value;
+};
+const debounceMs = readNumberFlag('--debounce', 2000);
+const positional = [];
+for (let i = 0; i < args.length; i++) {
+    const arg = args[i];
+    if (arg === undefined || arg === '--' || arg === '--clear' || arg === '--watch')
+        continue;
+    if (arg === '--debounce') {
+        i++;
+        continue;
+    }
+    positional.push(arg);
+}
+const folder = positional[0];
+const collectionArg = positional[1] ?? 'chunks';
+if (!folder || positional.length > 2) {
+    console.error('Usage: npm run ingest -- <folder> [collection] -- [--clear] [--watch] [--debounce <ms>]');
+    process.exit(1);
+}
+let collection;
+try {
+    collection = normalizeCollectionName(collectionArg);
+}
+catch (e) {
+    console.error(e.message);
+    process.exit(1);
+}
+const startedAt = Date.now();
+const resolvedFolder = resolve(folder);
+let lastLoggedFile = '';
+try {
+    const result = await ingestFolder(collection, resolvedFolder, {
+        clearFirst,
+        onProgress: (event) => {
+            const fileLabel = event.currentFile ? basename(event.currentFile) : '';
+            if (fileLabel && fileLabel !== lastLoggedFile) {
+                lastLoggedFile = fileLabel;
+                console.error(`[ingest] ${event.processedFiles}/${event.totalFiles} ${fileLabel}`);
+            }
+            if (event.status === 'starting') {
+                console.error(`[ingest] collection=${collection} folder=${resolvedFolder}`);
+            }
+        },
+    });
+    await recoverPendingIndexes({ collection });
+    console.error(`[ingest] done files=${result.files.length} chunks=${result.ingested} errors=${result.errors.length} elapsedMs=${Date.now() - startedAt}`);
+    console.log(JSON.stringify({ collection, ...result }, null, 2));
+    if (!watchMode) {
+        process.exit(result.errors.length > 0 ? 2 : 0);
+    }
+    const stopRecovery = startIndexRecovery();
+    const watcher = watchFolder(collection, resolvedFolder, {
+        debounceMs,
+        onBatch: (files, batchResult, error) => {
+            if (error) {
+                console.error(`[watch] re-index failed: ${error.message}`);
+                return;
+            }
+            const names = files.map((file) => basename(file)).join(', ');
+            console.error(`[watch] re-indexed ${files.length} file(s) (${names}) chunks=${batchResult?.ingested ?? 0} errors=${batchResult?.errors.length ?? 0}`);
+        },
+    });
+    console.error(`[watch] watching ${resolvedFolder} (debounce ${debounceMs}ms) — Ctrl+C to stop`);
+    const shutdown = async () => {
+        stopRecovery();
+        console.error('[watch] stopping…');
+        await watcher.close();
+        process.exit(0);
+    };
+    process.on('SIGINT', shutdown);
+    process.on('SIGTERM', shutdown);
+}
+catch (e) {
+    console.error(`[ingest] failed: ${e.message}`);
+    process.exit(1);
+}

package/dist/src/ingest.js

@@ -0,0 +1,257 @@
+import { readdir, readFile } from 'fs/promises';
+import { join, relative, resolve } from 'path';
+import { parseConversationGroups, detectProvider } from './parser.js';
+import { replaceImportedFiles } from './store.js';
+const SKIP_FILES = new Set([
+    'users.json',
+    'projects.json',
+    'user.json',
+    'message_feedback.json',
+    'shared_conversations.json',
+    'sora.json',
+    'applet_access_history.json',
+].map((n) => n.toLowerCase()));
+// Name-level filter shared with watch mode (which only has a path, no dirent).
+export const isExportFileName = (name) => {
+    const lowerName = name.toLowerCase();
+    if (SKIP_FILES.has(lowerName))
+        return false;
+    if (!lowerName.endsWith('.json') && /^file[-_]/.test(lowerName))
+        return false;
+    return lowerName.endsWith('.json') || !name.includes('.');
+};
+const isPotentialExportFile = (entry) => {
+    return entry.isFile() && isExportFileName(entry.name);
+};
+const getCanonicalExportKey = (fileName) => {
+    const lower = fileName.replace(/\\/g, '/').toLowerCase();
+    return lower.endsWith('.json') ? lower.slice(0, -5) : lower;
+};
+const collectExportEntries = async (folderPath) => {
+    const entries = [];
+    const walk = async (currentDir) => {
+        const dirEntries = await readdir(currentDir, { withFileTypes: true });
+        for (const entry of dirEntries) {
+            const fullPath = join(currentDir, entry.name);
+            if (entry.isDirectory()) {
+                await walk(fullPath);
+            }
+            else if (isPotentialExportFile(entry)) {
+                entries.push({
+                    name: entry.name,
+                    relativePath: relative(folderPath, fullPath),
+                    fullPath,
+                });
+            }
+        }
+    };
+    await walk(folderPath);
+    return entries;
+};
+const chooseExportFiles = (entries) => {
+    const byKey = new Map();
+    for (const entry of entries) {
+        const key = getCanonicalExportKey(entry.relativePath || entry.name);
+        const existing = byKey.get(key);
+        const isJson = entry.name.toLowerCase().endsWith('.json');
+        if (!existing || (isJson && !existing.name.toLowerCase().endsWith('.json'))) {
+            byKey.set(key, entry);
+        }
+    }
+    return [...byKey.values()].map((entry) => entry.fullPath);
+};
+export const listExportFiles = async (folderPath) => {
+    const resolved = resolve(folderPath);
+    const entries = await collectExportEntries(resolved);
+    return chooseExportFiles(entries);
+};
+export const ingestFolder = async (collection, folderPath, opts = {}) => {
+    const resolved = resolve(folderPath);
+    opts.signal?.throwIfAborted();
+    let entries;
+    try {
+        entries = await collectExportEntries(resolved);
+    }
+    catch (e) {
+        return {
+            conversations: 0,
+            ingested: 0,
+            totalTokens: 0,
+            files: [],
+            errors: [`Folder error: ${e.message}`],
+            providers: {},
+            elapsedMs: 0,
+        };
+    }
+    const files = chooseExportFiles(entries);
+    return ingestFilesInternal(collection, files, opts, opts.clearFirst === true);
+};
+// Ingest an explicit list of export files (watch mode re-indexes just the
+// files that changed). `clearFirst` is intentionally ignored here — clearing
+// belongs to whole-folder ingests.
+export const ingestFiles = async (collection, files, opts = {}) => ingestFilesInternal(collection, files, opts, false);
+const ingestFilesInternal = async (collection, files, opts, clearFirst) => {
+    const { onProgress = () => { } } = opts;
+    opts.signal?.throwIfAborted();
+    const errors = [];
+    const totalFiles = files.length;
+    let processedFiles = 0;
+    let totalChunks = 0;
+    let totalTokens = 0;
+    let totalConversations = 0;
+    const ingestedFiles = [];
+    const stagedFiles = [];
+    const skippedFiles = [];
+    const providers = {};
+    const startTime = Date.now();
+    onProgress({
+        status: 'starting',
+        totalFiles,
+        processedFiles: 0,
+        totalChunks: 0,
+        totalTokens: 0,
+        elapsedMs: 0,
+    });
+    for (const filePath of files) {
+        opts.signal?.throwIfAborted();
+        try {
+            onProgress({
+                status: 'progress',
+                phase: 'reading',
+                totalFiles,
+                processedFiles,
+                currentFile: filePath,
+                totalChunks,
+                totalTokens,
+                elapsedMs: Date.now() - startTime,
+                errorsCount: errors.length,
+                providers,
+            });
+            const raw = await readFile(filePath, 'utf-8');
+            opts.signal?.throwIfAborted();
+            // Parse once, use for both detection and extraction
+            let jsonData;
+            try {
+                jsonData = JSON.parse(raw);
+            }
+            catch {
+                errors.push(`${filePath}: Invalid JSON`);
+                processedFiles++;
+                continue;
+            }
+            const provider = detectProvider(jsonData);
+            providers[provider] = (providers[provider] || 0) + 1;
+            const parsed = parseConversationGroups(jsonData);
+            if (parsed.error) {
+                errors.push(`${filePath}: ${parsed.error}`);
+                processedFiles++;
+                continue;
+            }
+            if (parsed.conversations.length === 0) {
+                skippedFiles.push(filePath);
+                processedFiles++;
+                continue;
+            }
+            totalConversations += parsed.conversations.length;
+            onProgress({
+                status: 'progress',
+                phase: clearFirst ? 'reading' : 'embedding',
+                progressPercent: clearFirst ? (20 * processedFiles) / Math.max(totalFiles, 1) : undefined,
+                totalFiles,
+                processedFiles,
+                currentFile: filePath,
+                totalChunks,
+                totalTokens,
+                elapsedMs: Date.now() - startTime,
+                errorsCount: errors.length,
+                providers,
+            });
+            const imported = { sourceFile: filePath, provider, conversations: parsed.conversations };
+            if (clearFirst)
+                stagedFiles.push(imported);
+            else {
+                const previousTokens = totalTokens;
+                totalChunks += await replaceImportedFiles(collection, [imported], {
+                    signal: opts.signal,
+                    onEmbeddingProgress: (done, total, tokens) => {
+                        totalTokens = previousTokens + tokens;
+                        onProgress({
+                            status: 'progress',
+                            phase: 'embedding',
+                            totalFiles,
+                            processedFiles,
+                            currentFile: filePath,
+                            totalChunks,
+                            totalTokens,
+                            elapsedMs: Date.now() - startTime,
+                            embeddingDone: done,
+                            embeddingTotal: total,
+                            progressPercent: (95 * (processedFiles + done / Math.max(total, 1))) / Math.max(totalFiles, 1),
+                        });
+                    },
+                });
+                ingestedFiles.push(filePath);
+            }
+        }
+        catch (e) {
+            if (opts.signal?.aborted)
+                throw opts.signal.reason;
+            errors.push(`${filePath}: ${e.message}`);
+        }
+        processedFiles++;
+        onProgress({
+            status: 'progress',
+            phase: clearFirst ? 'reading' : undefined,
+            progressPercent: ((clearFirst ? 20 : 95) * processedFiles) / Math.max(totalFiles, 1),
+            totalFiles,
+            processedFiles,
+            currentFile: filePath,
+            totalChunks,
+            totalTokens,
+            elapsedMs: Date.now() - startTime,
+            errorsCount: errors.length,
+            providers,
+        });
+    }
+    const replacementSkipped = clearFirst && (errors.length > 0 || skippedFiles.length > 0 || stagedFiles.length === 0);
+    if (clearFirst && !replacementSkipped) {
+        try {
+            totalChunks = await replaceImportedFiles(collection, stagedFiles, {
+                clearFirst: true,
+                signal: opts.signal,
+                onEmbeddingProgress: (done, total, tokens) => {
+                    totalTokens = tokens;
+                    onProgress({
+                        status: 'progress',
+                        phase: 'embedding',
+                        totalFiles,
+                        processedFiles,
+                        totalChunks: done,
+                        totalTokens,
+                        elapsedMs: Date.now() - startTime,
+                        embeddingDone: done,
+                        embeddingTotal: total,
+                        progressPercent: 20 + (75 * done) / Math.max(total, 1),
+                    });
+                },
+            });
+            ingestedFiles.push(...stagedFiles.map((file) => file.sourceFile));
+        }
+        catch (error) {
+            if (opts.signal?.aborted)
+                throw opts.signal.reason;
+            errors.push(`Collection replacement: ${error.message}`);
+        }
+    }
+    return {
+        conversations: replacementSkipped ? 0 : totalConversations,
+        ingested: totalChunks,
+        totalTokens,
+        files: ingestedFiles,
+        errors,
+        ...(skippedFiles.length ? { skippedFiles } : {}),
+        ...(replacementSkipped ? { replacementSkipped: true } : {}),
+        providers,
+        elapsedMs: Date.now() - startTime,
+    };
+};

package/dist/src/load-env.js

@@ -0,0 +1,17 @@
+import { loadEnvFile } from 'node:process';
+import { dataPath } from './paths.js';
+/**
+ * Loads local development/runtime settings without overwriting variables that
+ * were explicitly provided by the parent process.
+ */
+export const loadThreadShelfEnv = (path = dataPath('env')) => {
+    try {
+        loadEnvFile(path);
+        return true;
+    }
+    catch (error) {
+        if (error.code === 'ENOENT')
+            return false;
+        throw error;
+    }
+};

package/dist/src/model-label.js

@@ -0,0 +1,15 @@
+// Keep provider IDs such as "google/gemini-2.0" intact while removing private
+// filesystem prefixes from legacy local-model values.
+export const portableModelLabel = (value) => {
+    if (!value)
+        return '';
+    const model = String(value).trim();
+    if (!/^[a-z]:[\\/]/i.test(model) && !model.startsWith('/') && !model.includes('\\')) {
+        return model;
+    }
+    return model
+        .replace(/\\/g, '/')
+        .split('/')
+        .at(-1)
+        .replace(/\.gguf$/i, '');
+};